#!/usr/bin/env node

require('isomorphic-fetch')
const cheerio = require('cheerio')
const Url = require('url')

const defaultOptions = {
  // throttle??
  // limit
  //
  includeExternal: true,
  fetchExternal: false,
}

// Make initial promise from startUrls
// Go to pages and make more promises [cascading]
// After all the promises are done return data.


function buildOutput(opt, callback) {
  const options = Object.assign({}, defaultOptions, opt)
  const output = {
    pages: {
      internal: {},
      external: {},
    }
  }
  const requests = {}

  const p = new Promise(function(resolve, reject) {
    if (options.startUrls && options.startUrls.length) {
      const promises = options.startUrls.map(function(startUrl) {
        requests[startUrl] = {}
        requests[startUrl].startTime = Date.now()
        return fetchUrl(startUrl, true)
      })
      return Promise.all(promises).then(function(d) {
        resolve(output)
      })
    }
  })

  // function fetchMultipleUrls(urls) {
  //   const promises = urls.map(function(url) {
  //     return fetchUrl(url, true)
  //   })
  //   return Promise.all(promises)
  // }

  function fetchUrl(url, internal) {
    url = url.split('#')[0]
    return fetch(url)
      .then(function(r) {
        requests[url].endTime = Date.now()
        requests[url].deltaTime = requests[url].endTime - requests[url].startTime
        // Add request stuff to data object.
        obj.status = r.status
        obj.lastModified = r.headers
          && r.headers._headers
          && r.headers._headers['last-modified']
          && r.headers._headers['last-modified'][0]
        return r.text()
      }).then(function(d) {

        const $ = cheerio.load(d)
        const links = $('a')
        const newPromises = []

        // Build output object.
        const obj = {}
        obj.linkCount = links.length
        obj.url = url
        obj.isInternal = internal
        obj.fetchTime = requests[url].deltaTime
        const where = internal ? 'internal' : 'external'
        // This sets the data key to output.
        output.pages[where][url] = obj

        if (internal) {
          links.each((i, link) => {
            const fullUrl = Url.resolve(url, link.attribs.href).split('#')[0]
            if (!requests[fullUrl]) {
              const fullUrlInternal = isInternal(url, fullUrl)
              if (fullUrlInternal || options.includeExternal) {
                requests[fullUrl] = {}
                if (fullUrlInternal || options.fetchExternal) {
                  requests[fullUrl].startTime = Date.now()
                  // console.log(fullUrl, fullUrlInternal);
                  newPromises.push(fetchUrl(fullUrl, fullUrlInternal))
                } else {
                  output.pages.external[fullUrl] = {}
                }
              }
            }
          })
        }



        if (newPromises.length > 0) {
          return Promise.all(newPromises)
        }

        return output
      }).catch(function(e) {
        console.log(e);
      })
  }

  return p
}

function isInternal(baseUrl, url) {
  const hostname = Url.parse(baseUrl).hostname
  return hostname === Url.parse(url).hostname
}


// function processUrl(url, requests) {
//   // get url and check against cache
//   // get output and determine if internal / external
// }
// Add to map


buildOutput({
  startUrls: [
    'http://blog.timscanlin.net'
  ]
}, function(err, o) {
  console.log(o);
}).then(function(data) {
  // console.log(data);
  process.stdout.write(JSON.stringify(data))
})

@@ -134,24 +134,8 @@
 = %7B%0A
-  // throttle??%0A
   //
@@ -141,16 +141,16 @@
 / limit%0A
+
   //%0A  i
@@ -1182,24 +1182,43 @@
 lit('#')%5B0%5D%0A
+    const obj = %7B%7D%0A
     return f
@@ -1807,31 +1807,8 @@
 ct.%0A
-        const obj = %7B%7D%0A
     
@@ -2146,16 +2146,85 @@
 k) =%3E %7B%0A
+            const href = link.attribs.href%0A            if (href) %7B%0A  
         
@@ -2260,29 +2260,16 @@
 ve(url, 
-link.attribs.
 href).sp
@@ -2288,24 +2288,26 @@
             
+ 
+ 
 if (!request
@@ -2316,24 +2316,26 @@
 fullUrl%5D) %7B%0A
+  
             
@@ -2381,24 +2381,26 @@
 l, fullUrl)%0A
+  
             
@@ -2459,32 +2459,34 @@
                 
+  
 requests%5BfullUrl
@@ -2484,32 +2484,34 @@
 s%5BfullUrl%5D = %7B%7D%0A
+  
                 
@@ -2550,32 +2550,34 @@
 etchExternal) %7B%0A
+  
                 
@@ -2625,66 +2625,8 @@
 )%0A  
-                // console.log(fullUrl, fullUrlInternal);%0A
     
@@ -2692,16 +2692,18 @@
 ernal))%0A
+  
         
@@ -2729,32 +2729,34 @@
                 
+  
 output.pages.ext
@@ -2767,32 +2767,52 @@
 l%5BfullUrl%5D = %7B%7D%0A
+                  %7D%0A
                 
@@ -3398,27 +3398,33 @@
 p://
-blog.timscanlin.net
+developers.optimizely.com
 '%0A  

fix bug