package gov.nist.crawleval;

import com.google.common.base.Charsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import de.l3s.boilerpipe.extractors.KeepEverythingExtractor;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.shingle.ShingleAnalyzerWrapper;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;

public class CompareFiles {

    @Option(name="-h", usage="Print out help")
    public boolean show_help = false;

    @Option(name="-n", usage="Number of bands in hash table (default 16)")
    public int num_bands = 16;

    @Argument
    public List<String> files = new ArrayList<String>();


    public final int num_hashes = 256;

    public static void main(String[] args) throws IOException {
        CompareFiles minhash = new CompareFiles();
        minhash.do_it(args);
    }

    public String get_string(Map map, String key) {
        Object val = map.get(key);
        if (val instanceof String && val != null)
            return (String)val;
        else
            return null;
    }

    protected Analyzer analyzer = new ShingleAnalyzerWrapper(new SimpleAnalyzer(), 9);
    protected MinHasher hasher = null;
    protected HashFunction murmur = Hashing.murmur3_32();
    protected HashFunction sha256 = Hashing.sha256();

    public int[] docToMinHashes(String raw_html) throws Exception {
        HashSet<Integer> doc = new HashSet<Integer>();
        String content_to_parse = KeepEverythingExtractor.INSTANCE.getText(raw_html);
        TokenStream ts = analyzer.tokenStream("extracted_text", content_to_parse);
        CharTermAttribute cattr = ts.addAttribute(CharTermAttribute.class);
        ts.reset();
        while (ts.incrementToken()) {
            doc.add(murmur.hashString(cattr.toString(), Charsets.UTF_8).asInt());
        }
        ts.close();
        if (hasher == null)
            hasher = new MinHasher(num_hashes);
        return hasher.hash(doc);

    }

    public void do_it(String[] args) throws IOException {
        CmdLineParser argparser = new CmdLineParser(this);
        try {
            argparser.parseArgument(args);
            if (files.isEmpty()) {
                throw new CmdLineException(argparser, "No files given");
            }

        } catch (CmdLineException cle) {
            System.err.println(cle.getMessage());
            argparser.printUsage(System.err);
            System.err.println();
        }
        if (show_help) {
            argparser.printUsage(System.out);
            System.out.println();
            System.exit(1);
        }

        System.err.println("Pass 1: Indexing using LSH");
        int count = 0;
        LSH lsh = new LSH(num_hashes, num_bands);

        for (String filename : files) {
            try {
                String raw = new String(Files.readAllBytes(Paths.get(filename)), StandardCharsets.UTF_8);

                int[] signatures = docToMinHashes(raw);
                lsh.insert(filename, signatures);

            } catch (IllegalArgumentException parse_err) {
                System.err.println("Error parsing " + filename);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        System.err.println("Pass 2: Querying near-duplicates");
        count = 0;
        HashMap<String, String> url2sha = new HashMap<>();

        for (String filename : files) {
            try {
                String raw = new String(Files.readAllBytes(Paths.get(filename)), StandardCharsets.UTF_8);

                int[] sigs = docToMinHashes(raw);
                Set<String> lsh_dupes = lsh.query(sigs);
                for (String d : lsh_dupes)
                    if (!d.equals(filename))
                        System.out.println(filename + " " + d);
            } catch (IllegalArgumentException parse_err) {
                System.err.println("Error parsing " + filename);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

@@ -949,27 +949,122 @@
  of 
-bands in hash table
+hash entries (default 256)%22)%0A    int num_hashes = 256;%0A%0A    @Option(name=%22-t%22, usage=%22Jaccard similarity threshold
  (de
@@ -1073,18 +1073,19 @@
 ult 
-16
+0.9
 )%22)%0A    
 publ
@@ -1084,33 +1084,38 @@
     
-public int num_bands = 16
+double jaccard_threshold = 0.9
 ;%0A%0A 
@@ -1189,49 +1189,8 @@
 );%0A%0A
-%0A    public final int num_hashes = 256;%0A%0A
     
@@ -1296,24 +1296,24 @@
 areFiles();%0A
+
         minh
@@ -1340,222 +1340,8 @@
  %7D%0A%0A
-    public String get_string(Map map, String key) %7B%0A        Object val = map.get(key);%0A        if (val instanceof String && val != null)%0A            return (String)val;%0A        else%0A            return null;%0A    %7D%0A%0A
     
@@ -2975,17 +2975,25 @@
 es, 
-num_bands
+jaccard_threshold
 );%0A%0A

Update arguments to use threshold and number of hashes. Remove get_string dead code.