// use simplecrawler
var Crawler = require('simplecrawler');

// use form-data and http to connect with CollaMine servers
var FormData = require('form-data');
var http = require('http');

// start crawling
var crawler = Crawler.crawl("http://forums.hardwarezone.com.sg/money-mind-210/");

crawler.interval = 1000;
crawler.timeout = 10000;

var conditionID = crawler.addFetchCondition(function(parsedURL) {
  return parsedURL.path.match(/\.html$/i); //&& (!try_collamine(parsedURL.path));
});

crawler.on("fetchstart", function(queueItem) {
  console.log("Fetching", queueItem.url);
  try_collamine(queueItem.url);
});

crawler.on("fetchcomplete", function(queueItem, responseBuffer, response) {
  console.log("Completed fetching resource:", queueItem.url);
  // if source == original then upload to collamine
  // insert to db
  // responseBuffer.toString();
});

// try downloading the content from CollaMine servers
function try_collamine(url) {
  var COLLAMINE_DOWNLOAD_URL = {
    host: '172.31.22.135',
    port: '9001',
    path: '/download/html/' + encodeURIComponent(url)
  };

  http.request(COLLAMINE_DOWNLOAD_URL, function(response) {
    var str = '';
    //another chunk of data has been recieved, so append it to `str`
    response.on('data', function (chunk) {
      str += chunk;
    });

    //the whole response has been recieved, so we just print it out here
    response.on('end', function () {
      console.log(str);
    });
  }).end();
}

// function upload_to_collamine() {
//   var form = new FormData();

//   http.request('http://nodejs.org/images/logo.png', function(response) {
//     form.append('my_field', 'my value');
//     form.append('my_buffer', new Buffer(10));
//     form.append('my_logo', response);
//   });
// }
@@ -337,26 +337,37 @@
 0;%0A%0A
-var conditionID = 
+// only fetch html documents%0A
 craw
@@ -456,52 +456,70 @@
 /i);
- //&& (!try_collamine(parsedURL.path));%0A%7D);%0A
+%0A%7D);%0A%0A// before fetch starts, check if doc exists in collamine
 %0Acra
@@ -605,16 +605,32 @@
 .url);%0A 
+ var collamine =
  try_col
@@ -652,16 +652,125 @@
 m.url);%0A
+  console.log(queueItem.fetched)%0A  if (collamine != 'not found') %7B%0A  %09console.log('get from collamine');%0A  %7D%0A
 %7D);%0A%0Acra
@@ -1317,16 +1317,17 @@
 ;%0A    //
+ 
 another 
@@ -1459,16 +1459,17 @@
 %0A%0A    //
+ 
 the whol
@@ -1495,43 +1495,13 @@
  rec
-i
 e
+i
 ved
-, so we just print it out here
 %0A   
@@ -1544,24 +1544,18 @@
     
-console.log(
+return 
 str
-)
 ;%0A  

add fetch condition