package jobs;

import com.google.common.base.Stopwatch;
import java.util.List;
import java.util.concurrent.TimeUnit;
import models.Citation;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import play.Logger;
import play.jobs.Job;
import play.vfs.VirtualFile;

public class LuceneIndexing extends Job {

    private final static int STEP = 100000;

    @Override
    public void doJob() throws Exception {

        Logger.info("Indexing started...");

//        Analyzer analyzer = new CustomStandardAnalyzer(Version.LUCENE_47);
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
        ShingleAnalyzerWrapper shingleAnalyzer = new ShingleAnalyzerWrapper(analyzer, 2, 3);

        Directory directory = FSDirectory.open(VirtualFile.fromRelativePath("/lucene").getRealFile());
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, shingleAnalyzer);
        config.setRAMBufferSizeMB(1024);
        IndexWriter iwriter = new IndexWriter(directory, config);
        
        //Iterate over the citations by packs of 1000
        //The total number as now is: 23772097
        //long totalCitations = Citation.count();
        int totalCitations = 23772097;
        
        //Add time information for when the data is fetched from the database
        for (int i = 0; i < totalCitations; i += STEP) {

            Logger.info("i: " + i + "/" + totalCitations);
            Stopwatch stopwatchdb = Stopwatch.createUnstarted();
            stopwatchdb.start();
            List<Citation> citations = Citation.all().from(i).fetch(STEP);
            stopwatchdb.stop();
            Logger.info("Time to query the DB: " + stopwatchdb.elapsed(TimeUnit.MILLISECONDS));

            for (Citation citation : citations) {

                Document doc = new Document();
//                if (citation.abstractText != null) {
//                    doc.add(new Field("abstract", citation.abstractText, TextField.TYPE_STORED));
//                }

                if (citation.title != null) {
                    doc.add(new Field("title", citation.title, TextField.TYPE_STORED));
                }
                doc.add(new Field("date", DateTools.dateToString(citation.created, DateTools.Resolution.MINUTE), TextField.TYPE_STORED));
                iwriter.addDocument(doc);
            }
        }

        iwriter.close();

        Logger.info("index done.");
    }
}

@@ -1368,25 +1368,17 @@
 onfig);%0A
-        
 %0A
+
         
@@ -1559,24 +1559,16 @@
 772097;%0A
-        
 %0A       
@@ -1637,16 +1637,24 @@
 database
+        
 %0A       
@@ -2052,13 +2052,8 @@
 nit.
-MILLI
 SECO
@@ -2056,24 +2056,129 @@
 SECONDS));%0A%0A
+            Stopwatch stopwatchindex = Stopwatch.createUnstarted();%0A            stopwatchindex.start();%0A%0A
             
@@ -2771,16 +2771,16 @@
 t(doc);%0A
-
         
@@ -2777,32 +2777,182 @@
 ;%0A            %7D%0A
+            %0A            stopwatchindex.stop();%0A            Logger.info(%22Time to index the documents: %22 + stopwatchindex.elapsed(TimeUnit.SECONDS));%0A%0A
         %7D%0A%0A     

update optimisation