var WebSocket = require('ws');
var request = require('request');
var argv = require('minimist')(process.argv.slice(2));
var ws = new WebSocket('ws://127.0.0.1:8000/');
var level = require('level');
var crypto = require("crypto");
var fs = require('fs');
var util = require('util');

// few defaults
argv._[0] = argv._[0] || 'example';
argv._[1] = argv._[1] || '.*';

var db = level(argv._[0] + '.db');
var scope = new RegExp(".*(" + argv._[1].replace(/\./g, '\\\.') + ").*");

console.log("SCOPE: " + scope)
var inScope = function(url){
  return scope.test(url);
}

var hash = function(input){
      return crypto.createHash('sha1').update(input).digest('hex').toString();
}

var history = function(url, cb){
  var key = hash(url);
  db.get(key, function(err, isNew){
    if(!err && isNew === "true"){
      cb = function(){};
      return cb();
    }
    else if (inScope(url)){
      return db.put(key, "true", function(err, val){
        return cb();
      })
    }
  })
};


ws.on('open', function() {
  console.log('opened connection to ws://localhost:8000/');
});

ws.on('message', function(data, flags) {
  var obj = JSON.parse(data);
  if (obj.messageType === 'request' && typeof obj.headers.Cookie !== 'undefined') {
    history(obj.url, function(){
    var req = {
        url: obj.url,
        method: obj.method,
        headers: stripCookies(obj.headers),
        followRedirect: false,
        strictSSL: false,
    };
    if (['patch', 'post', 'put'].indexOf(obj.method.toLowerCase()) !== -1 && obj.body.length > 0)  {
        req.body = new Buffer(obj.body);
    }
    request(req, function(err, resp, body) {
        fs.appendFileSync(argv._[0] + '.rpt', '\n' + resp.statusCode + " Hash: " + hash(resp.body) + " " + req.method + " " + obj.url);
        fs.appendFileSync(argv._[0] + '.log', util.inspect({ "Code": resp.statusCode, "URL":req.url, "Hash": hash(body), "Request": req, "Response": resp}))
        if (resp.statusCode < 300 && !err ) {
        fs.appendFileSync(argv._[0] + '.issues', '-'+ "\nWARNING: " + hash(resp.body) + '\nWARNING: ' + resp.statusCode + " " + req.method + " " + obj.url)
        console.log('-'+ "\nWARNING: " + hash(resp.body) + '\nWARNING: ' + resp.statusCode + " " + req.method + " " + obj.url);
        }
    });
    });
  }
});

ws.on('error', function(err) {
  console.log(err);
  process.exit(1);
});

function stripCookies(headers) {
    delete headers.Cookie;
    return headers;
}

@@ -500,16 +500,17 @@
 + scope)
+;
 %0Avar inS
@@ -554,24 +554,25 @@
 test(url);%0A%7D
+;
 %0A%0Avar hash =
@@ -669,16 +669,17 @@
 ing();%0A%7D
+;
 %0A%0Avar hi
@@ -958,16 +958,17 @@
       %7D)
+;
 %0A    %7D%0A 
@@ -970,16 +970,17 @@
   %7D%0A  %7D)
+;
 %0A%7D;%0A%0A%0Aws
@@ -1916,16 +1916,17 @@
  resp%7D))
+;
 %0A       
@@ -2119,16 +2119,17 @@
 obj.url)
+;
 %0A       

Add some semicolons