#!/usr/bin/env node
/**
 * https://www.mediawiki.org/wiki/citoid
 */

var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var request = require('request');
var urlParse = require('url');


/**
 * Currently picks out contents of <title> tag only
 * callback runs on list of json objs (var body)
 * @param  {String}   url      url to scrape
 * @param  {Function} callback callback function
 */

var scrapeXpath = function(url, callback){
	request(
		{
			url: url, 
			headers: {'user-agent': 'Mozilla/5.0'},
			followAllRedirects: true 
		}, function(error, response, html){

			var doc, body, titleValue,
				json = {itemType: 'webpage'};

			if (error || !response) {
				json['url'] = url;
				json['title'] = url;
				body = [json];
				callback(body);
				return;
			}

			try{
				doc = new dom().parseFromString(html);
			}
			catch (e){
				console.log(e);
			}

			try {
				titleValue = xpath.select("//title/text()", doc).toString();
			}
			catch (e){
				console.log(e);
			}

			parsedUrl = response.request.uri ? response.request.uri : urlParse.parse(url);
			json['url'] = url;

			d = new Date();
			json['accessDate'] = d.toDateString();

			json['title'] = titleValue ? titleValue : url;

			if (titleValue && parsedUrl && parsedUrl.hostname) {
				json['publicationTitle'] = parsedUrl.hostname;
			}

			body = [json];

			callback(body);
	});
};

var scrape = scrapeXpath;

if (require.main === module) {
	var sampleUrl = 'http://example.com';
	console.log('scrape fcn running on sample url:'+sampleUrl);
	scrape(sampleUrl, function(body){
		console.log(body);
	});
}

module.exports = {
	scrape: scrape
};
@@ -67,77 +67,8 @@
 */%0A%0A
-var xpath = require('xpath');%0Avar dom = require('xmldom').DOMParser;%0A
 var 
@@ -128,16 +128,50 @@
 'url');%0A
+var cheerio = require('cheerio');%0A
 %0A%0A/**%0A *
@@ -378,21 +378,16 @@
 r scrape
-Xpath
  = funct
@@ -406,16 +406,336 @@
 lback)%7B%0A
+%0A%09var $;%0A%0A%09function getTitle() %7B%0A%0A%09%09var title;%0A%0A%09%09// Try to get title from itemprop=%22heading%22%0A%09%09title = $('*%5Bitemprop~=%22headline%22%5D').first().text();%0A%09%09if (title) %7B return title; %7D%0A%0A%09%09// Try to get title from %3Ctitle%3E tag%0A%09%09title = $('title').first().text();%0A%09%09if (title) %7B return title; %7D%0A%0A%09%09// Default%0A%09%09return url;%0A%09%7D%0A%0A
 %09request
@@ -876,35 +876,8 @@
 var 
-doc, body, titleValue,%0A%09%09%09%09
 json
@@ -899,16 +899,38 @@
 webpage'
+, url: url, title: url
 %7D;%0A%0A%09%09%09i
@@ -958,75 +958,8 @@
 ) %7B%0A
-%09%09%09%09json%5B'url'%5D = url;%0A%09%09%09%09json%5B'title'%5D = url;%0A%09%09%09%09body = %5Bjson%5D;%0A
 %09%09%09%09
@@ -963,28 +963,30 @@
 %09%09%09callback(
-body
+%5Bjson%5D
 );%0A%09%09%09%09retur
@@ -1010,39 +1010,24 @@
 %09%09%09%09
-doc = new dom().parseFromString
+$ = cheerio.load
 (htm
@@ -1069,371 +1069,308 @@
 log(
-e);%0A%09%09%09%7D%0A%0A%09%09%09try %7B%0A%09%09%09%09titleValue = xpath.select(%22//title/text()%22, doc).toString();%0A%09%09%09%7D%0A%09%09%09catch (e)%7B%0A%09%09%09%09console.log(e);%0A%09%09%09%7D%0A%0A%09%09%09parsedUrl = response.request.uri ? response.request.uri : urlParse.parse(url);%0A%09%09%09json%5B'url'%5D = url;%0A%0A%09%09%09d = new Date();%0A%09%09%09json%5B'accessDate'%5D = d.toDateString();%0A%0A%09%09%09json%5B'title'%5D = titleValue ? titleValue : url;%0A%0A%09%09%09if (titleValu
+'Could not load document: ' + e);%0A%09%09%09%09callback(%5Bjson%5D);%0A%09%09%09%7D%0A%0A%09%09%09json.title = getTitle();%0A%0A%09%09%09// Access date on format YYYY-MM-DD%0A%09%09%09json.accessDate = (new Date()).toISOString().substring(0, 10);%0A%0A%09%09%09var parsedUrl = response.request.uri ? response.request.uri : urlParse.parse(url);%0A%0A%09%09%09if (json.titl
 e &&
@@ -1417,10 +1417,9 @@
 json
-%5B'
+.
 publ
@@ -1434,10 +1434,8 @@
 itle
-'%5D
  = p
@@ -1463,27 +1463,8 @@
 %09%7D%0A%0A
-%09%09%09body = %5Bjson%5D;%0A%0A
 %09%09%09c
@@ -1471,20 +1471,22 @@
 allback(
-body
+%5Bjson%5D
 );%0A%09%7D);%0A
@@ -1493,35 +1493,8 @@
 %7D;%0A%0A
-var scrape = scrapeXpath;%0A%0A
 if (

Improve scraping