require("dotenv").load({silent: true});
var async = require("async");
var aws = require("aws-lib");
var DbConnector = require("./graphdb_connector");
var log = require("./log");
var path = require("path");
var priceAsin = require("./price_connector");
var RateLimiter = require("limiter").RateLimiter;
var util = require("util");

var fail = function() {
	throw new Error("Missing required env var");
};

function CrawlQueue(options) {
	if (!options) {
		options = {};
	}
	this.maxCrawlDepth = options.maxCrawlDepth || 2;
	this.nodeCount = 0;
	log.debug({}, "maxCrawlDepth: " + this.maxCrawlDepth);

	var keyId = process.env.AMZN_ACCESS_KEY_ID || fail();
	var keySecret = process.env.AMZN_ACCESS_KEY_SECRET || fail();
	var associateTag = process.env.AMZN_ASSOCIATE_TAG || fail();
	var amazonServiceHost = process.env.AMZN_SERVICE_HOST || "webservices.amazon.co.uk";

	this.prodAdv = aws.createProdAdvClient(keyId, keySecret, associateTag, { host: amazonServiceHost});
	this.limiter = new RateLimiter(1, "second");
	this.db = new DbConnector();
}

CrawlQueue.prototype.throttledSimilarityLookup = function(asin, callback) {
	var self = this;
	this.limiter.removeTokens(1, function(err) {
		if (err) {
			return callback(err);
		}
		self.prodAdv.call("SimilarityLookup", { ItemId: asin }, callback);
	});
};

CrawlQueue.prototype.throttledPriceLookup = function(asin, callback) {
	this.limiter.removeTokens(1, function(err) {
		if (err) {
			return callback(err);
		}
		priceAsin.fetch(asin, callback);
	});
};


CrawlQueue.prototype.alreadyCrawled = function(asin, callback) {
	var self = this;
	this.db.getBookNode(asin, function(err, node) {
		if (err) {
			return callback(err);
		}
		if (!node) {
			return callback(null, {crawled: false});
		}
		self.db.countOutgoingRecommendations(asin, function(err, result) {
			if (err) {
				return callback(err);
			}
			if (!result.outgoing) {
				return callback(null, {crawled: false});
			}
			callback(null, {crawled: true});
		});

	});
};

CrawlQueue.prototype.crawl = function(rootAsin, depth, callback) {

	this.nodeCount++;
	log.debug({current_depth: depth, parent_node: rootAsin}, "crawling");
	var self = this;
	depth += 1;
	if (depth > self.maxCrawlDepth) {
		log.debug({}, util.format("Reached depth %s, stop crawling", depth));
		return callback();
	}
	async.waterfall([
		function(cb) {
			self.throttledSimilarityLookup(rootAsin, cb);
		},
		function(similar, cb) {
			if (!similar.Items.Item) {
				log.warn({asin: rootAsin}, "Similar items list is empty");
				return cb();
			}
			log.debug({asin: rootAsin, child_count: similar.Items.Item.length}, "Start crawling children");
			async.each(similar.Items.Item, function(item, each_cb) {
				async.waterfall([
					function(cb) {
						//write into graph db
						self.throttledPriceLookup(item.ASIN, function(err, result) {
							if (err) {
								log.error({error: err.message}, "error looking up price for ASIN " + item.ASIN);
								// add to graph without price, so drop this error
							}
							if (result) {
								item.price = result.price;
								item.currency = result.currency;
							}
							self.db.createChildBookNodeAndRelations(rootAsin, item, function(err, result) {
							    log.debug({result: result}, "Finished creating node and relations");
								// drop result to not cause problems at next step
								cb(err);
							});
						});
					},
					function(cb) {
						self.crawl(item.ASIN, depth, cb);
					}
				], function(err) {
					log.debug({asin: item.ASIN}, "Finished walking child node");
					each_cb(err);
				});
			}, cb);
		}
	], function(err) {
		log.debug({asin: rootAsin}, "Finished crawling");
		callback(err);
	});
};

CrawlQueue.prototype.createNodeWithAsin = function(asin, callback) {
	var self = this;
	this.limiter.removeTokens(1, function(err) {
		if (err) {
			return callback(err);
		}
		self.prodAdv.call("ItemLookup", { ItemId: asin }, function(err, result) {
			self.db.createBookNode(result.Items.Item, callback);
		});
	});
};

CrawlQueue.prototype.keywordSearch = function(keyword, responseGroup, callback) {
	this.prodAdv.call("ItemSearch", { Keywords: keyword, ResponseGroup: responseGroup }, callback);
};

module.exports = CrawlQueue;

@@ -89,24 +89,58 @@
 %22aws-lib%22);%0A
+var config = requite(%22./config%22);%0A
 var DbConnec
@@ -173,24 +173,24 @@
 onnector%22);%0A
-
 var log = re
@@ -362,82 +362,8 @@
 );%0A%0A
-var fail = function() %7B%0A%09throw new Error(%22Missing required env var%22);%0A%7D;%0A%0A
 func
@@ -566,28 +566,28 @@
 keyId = 
-process.env.
+config.get(%22
 AMZN_ACC
@@ -596,25 +596,23 @@
 S_KEY_ID
- %7C%7C fail(
+%22, true
 );%0A%09var 
@@ -619,36 +619,36 @@
 keySecret = 
-process.env.
+config.get(%22
 AMZN_ACCESS_
@@ -657,25 +657,23 @@
 Y_SECRET
- %7C%7C fail(
+%22, true
 );%0A%09var 
@@ -687,28 +687,28 @@
 teTag = 
-process.env.
+config.get(%22
 AMZN_ASS
@@ -721,17 +721,15 @@
 _TAG
- %7C%7C fail(
+%22, true
 );%0A%09
@@ -756,20 +756,20 @@
 t = 
-process.env.
+config.get(%22
 AMZN
@@ -781,16 +781,18 @@
 ICE_HOST
+%22)
  %7C%7C %22web

Update crawl_queue.js