"use strict";

var request = require("request");
var cheerio = require("cheerio");
var mongoose = require("mongoose");
var async = require("async");
var crypto = require("crypto");


// how many days should we parse?
var daysToParse = 21;


// mongodb
var mongohost="localhost:27017";
var mongodb=process.env.WUWDB || "wuw";
var mongoConnection="mongodb://" + mongohost + "/" + mongodb;
// connect
mongoose.connect(mongoConnection);


// simple hash-algo to generate 12 byte long objectId
var hashCode = function(s){
    var hash = crypto.createHash('md5').update(s).digest('hex').substring(0, 12);
    return hash;
};

// parsing
var parse = function(html, cb) {
    // create models from our schemas
    var Lecture = require("./models/model_lecture");

    // init cheerio with our html
    var $ = cheerio.load(html);

    // get date for this day
    var curDate = $("h2").text().split(", ")[1];
    var curDateArr = curDate.split(".");
    var intDate = curDateArr[1] + "/" + curDateArr[0] + "/" + curDateArr[2];


    // parse each single lecture
    async.each($("tr").not(":first-child"), function(lectureLine, trcb) {

        // prepare docents
        var docents = [];
        $(lectureLine).children().eq(7).text().trim().split(" , ").forEach(function(docent){
            if(docent !== "") {
                docents.push(docent.replace(/.*Professor/g, "").replace(/ */g, ""));
            }
        });

        // prepare group & room (not really needed but more readable)
        var group = $(lectureLine).children().eq(6).text().trim();
        var room = $(lectureLine).children().eq(5).text().trim();

        // create Lecture from our Model
        var Lec = new Lecture();
        // set attributes
        Lec.lectureName = $(lectureLine).children().eq(3).text().trim();
        Lec.startTime = new Date(intDate + " " + $(lectureLine).children().eq(0).text().trim());
        Lec.endTime = new Date(intDate + " " + $(lectureLine).children().eq(1).text().trim());
        Lec.docents = docents;
        Lec.hashCode = hashCode(Lec.lectureName+curDate+Lec.startTime);
        Lec._id = mongoose.Types.ObjectId(Lec.hashCode);

        // create an object from our document
        var upsertData = Lec.toObject();
        // delete attributes to upsert
        delete upsertData.rooms;
        delete upsertData.groups;

        // lectures without a group/room are useless...
        if(group !== "" && room !== "") {
            // save lecture to db & call callback
            Lecture.update({ _id: Lec.id }, { $set: upsertData, $addToSet: { rooms: room, groups: group }  }, { upsert: true }, trcb);
        } else {
            trcb();
        }
    }, function() {
        // day done
        cb();
    });
};

// create urls including the date which we want to parse
var createUrls = function() {
    var urls = [];
    for(var i = 0; i < daysToParse; i++) {
        // get date to parse
        var today = new Date();
        today.setDate(today.getDate() + i);
        // assemble datestring & url
        var currDay = today.getDate() + "." + (today.getMonth()+1) + "." + today.getFullYear();
        var url = "https://lsf.hft-stuttgart.de/qisserver/rds?state=currentLectures&type=0&next=CurrentLectures.vm&nextdir=ressourcenManager&navigationPosition=lectures&currentLectures&breadcrumb=currentLectures&topitem=lectures&subitem=currentLectures&P.Print=&&HISCalendar_Date=" + currDay + "&asi=";
        urls.push(url);
    }
    return urls;
};

var startParser = function() {
    // connect to mongodb (if not already)
    if(mongoose.connection.readyState === 0) {
        mongoose.connect(mongoConnection);
    }

    // create model from our schema (needed for drop)
    var Lecture = require("./models/model_lecture");

    // create the urls to parse
    var urls = createUrls();

    console.log("started parsing of " + daysToParse + " days (this inlcuded)...");

    // drop current lecture collection to get a fresh result
    mongoose.connection.collections.lectures.drop(function(err) {
        if(err) { console.log(err); }

        console.log("dropped old \""+ Lecture.collection.name + "\" collection, lets start clean...");

        // parse every url (max. 5 parallel, dont fuck the lsf)
        async.eachLimit(urls, 5, function(url, cb) {
            request(url, function(error, response, html) {
                if(!error) {
                    // parse html with all lectures for choosen date
                    parse(html, cb);
                }
            });
        }, function() {
            // everything done
            mongoose.disconnect();
            console.log("...done!");
        });
    });
};

// start the magic
startParser();

module.exports = { startParser: startParser, parse: parse };

@@ -3835,15 +3835,57 @@
 og(%22
-started
+%5Cn * lsf parser started%5Cn%22);%0A    console.log(%22  *
  par
@@ -3893,11 +3893,8 @@
 ing 
-of 
 %22 + 
@@ -3917,306 +3917,48 @@
 days
- (this inlcuded)...
 %22);%0A
-%0A
     
-// drop current lecture collection to get a fresh result%0A    mongoose.connection.collections.lectures.drop(function(err) %7B%0A        if(err) %7B console.log(err); %7D%0A%0A        console.log(%22dropped old %5C%22%22+ Lecture.collection.name + %22%5C%22 collection, lets start clean...
+process.stdout.write(%22%5Cn   
 %22);%0A%0A
-    
     
@@ -3981,23 +3981,20 @@
 rl (
-max. 5 parallel
+rate-limited
 , do
@@ -4014,20 +4014,16 @@
 sf)%0A    
- 
-   
 async.ea
@@ -4055,28 +4055,24 @@
 (url, cb) %7B%0A
-    
         requ
@@ -4126,20 +4126,16 @@
         
-  
-  
 if(!erro
@@ -4135,28 +4135,24 @@
 f(!error) %7B%0A
-    
             
@@ -4220,20 +4220,16 @@
         
-  
-  
 parse(ht
@@ -4231,19 +4231,148 @@
 se(html,
+ function() %7B%0A                    // simple progress display%0A                    process.stdout.write(%22 *%22);%0A                   
  cb
+(
 );%0A     
@@ -4375,32 +4375,34 @@
                %7D
+);
 %0A            %7D);
@@ -4395,31 +4395,37 @@
            %7D
-);
 %0A    
+    %7D);%0A
     %7D, funct
@@ -4424,36 +4424,32 @@
 %7D, function() %7B%0A
-    
         // every
@@ -4463,28 +4463,24 @@
 one%0A        
- 
-   
 mongoose.dis
@@ -4498,20 +4498,16 @@
         
-    
 console.
@@ -4515,29 +4515,24 @@
 og(%22
-...done!%22);%0A        %7D
+%5Cn%5Cn  * done!%5Cn%22
 );%0A 

remove collection drop before importing data