package com.deckerchan.ml;

import com.deckerchan.ml.classifier.entities.Coordinate;
import com.deckerchan.ml.classifier.entities.Dimension;
import com.deckerchan.ml.classifier.entities.RealItemHDPoint;
import com.deckerchan.ml.classifier.entities.WordFrequencyBasedValueTable;
import com.deckerchan.ml.classifier.k.KClassifierBase;
import com.deckerchan.ml.classifier.k.KMeansClassifier;
import com.deckerchan.ml.io.Document;
import com.deckerchan.ml.io.EmailFormatDocument;
import com.deckerchan.ml.io.PureTextDocument;

import javax.print.Doc;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import static java.lang.System.out;

public class EntryPoint {
    public static void main(String[] args) throws Exception {


        ArrayList<Document> documents = Files
                .walk(Paths.get(args[0]))
                .filter(Files::isRegularFile)
                .map(EmailFormatDocument::new)
                .collect(Collectors.toCollection(ArrayList<Document>::new));

        documents.parallelStream().forEach(Document::calculateWordFrequencyTable);

        //Get total word frequency table

        WordFrequencyBasedValueTable totalTable = new WordFrequencyBasedValueTable();
        totalTable.mergeTable(documents.stream()
                .map(Document::getWordFrequencyBasedValueTable)
                .reduce(new WordFrequencyBasedValueTable(), WordFrequencyBasedValueTable::mergeTable).getSortedTableOrderByValue(400));


        //idf
        for (String word : totalTable.getWords()) {
            Long df = documents.stream().filter(document -> document.getWordFrequencyBasedValueTable().contains(word)).count();
            int totoalDocumentNumner = documents.size();
            Double idf = Math.log10(totoalDocumentNumner / df);
            documents.stream().filter(document -> document.getWordFrequencyBasedValueTable().getWords().contains(word))
                    .forEach(newDoc -> newDoc.getWordFrequencyBasedValueTable().put(word,newDoc.getWordFrequencyBasedValueTable().get(word)*idf));
        }

        List<Dimension> dimensionList = totalTable
                .entrySet().parallelStream()
                .map(stringLongEntry -> new Dimension(stringLongEntry.getKey()))
                .collect(Collectors.toList());


        List<RealItemHDPoint> documentPoints = documents.parallelStream()
                .map(document -> {
                    Coordinate initialDimensionMap = new Coordinate(dimensionList);


                    for (String word : document.getWordFrequencyBasedValueTable().keySet()) {
                        initialDimensionMap.set(word, document.getWordFrequencyBasedValueTable().get(word));
                    }

                    out.printf("Finish build hyper point for document %s%n", document.getFileName());
                    return new RealItemHDPoint<>(document, initialDimensionMap);
                })
                .collect(Collectors.toList());

        KClassifierBase classifier = new KMeansClassifier(Integer.valueOf(args[1]), documentPoints);

        classifier.calculate(50);

        classifier.report();


        System.exit(0);


    }
}

@@ -473,78 +473,8 @@
 nt;%0A
-import com.deckerchan.ml.io.PureTextDocument;%0A%0Aimport javax.print.Doc;
 %0Aimp
@@ -1555,43 +1555,39 @@
 f = 
-documents.stream().filter(d
+0L;%0A%0A            for (D
 ocument 
 -%3E d
@@ -1574,34 +1574,37 @@
   for (Document 
--%3E
+doc :
  document.getWor
@@ -1588,32 +1588,59 @@
 t doc : document
+s) %7B%0A                if(doc
 .getWordFrequenc
@@ -1658,16 +1658,27 @@
 Table().
+getWords().
 contains
@@ -1688,17 +1688,68 @@
 rd))
-.count();
+%7B%0A                    df++;%0A                %7D%0A            %7D%0A
 %0A   
@@ -2075,16 +2075,17 @@
 ut(word,
+ 
 newDoc.g
@@ -2130,9 +2130,11 @@
 ord)
-*
+ * 
 idf)

FIx idf divide 0.