'use strict'

var fs = require('fs')
var http = require('http')
var bail = require('bail')
var concat = require('concat-stream')
var unified = require('unified')
var parse = require('rehype-parse')
var selectAll = require('hast-util-select').selectAll
var toString = require('hast-util-to-string')

var endpoint =
  'http://www.readabilityformulas.com/articles/dale-chall-readability-word-list.php'

http.get(endpoint, onresponse)

function onresponse(res) {
  res.pipe(concat(onconcat)).on('error', bail)
}

function onconcat(buf) {
  var tree = unified()
    .use(parse)
    .parse(buf)

  var values = selectAll('.main_container_table_three td', tree)
    .map(toString)
    .join(' ')
    .trim()
    .split(/\s+/g)
    .filter(Boolean)
    .map(function(value) {
      return value.toLowerCase()
    })
    .sort()
    .filter(function(value, index, all) {
      return !all.includes(value, index + 1)
    })

  fs.writeFile('index.json', JSON.stringify(values, 0, 2) + '\n', bail)
}

@@ -38,16 +38,17 @@
 var http
+s
  = requi
@@ -55,16 +55,17 @@
 re('http
+s
 ')%0Avar b
@@ -316,16 +316,17 @@
 %0A  'http
+s
 ://www.r
@@ -400,16 +400,17 @@
 p'%0A%0Ahttp
+s
 .get(end
@@ -876,39 +876,36 @@
 urn 
-!
 all.in
-cludes
+dexOf
 (value
-,
+) ===
  index
- + 1)
 %0A   

Fix build script