@@ -0,0 +1,2768 @@
+package edu.neu.ccs.pyramid.experiment;%0A%0Aimport edu.neu.ccs.pyramid.configuration.Config;%0Aimport edu.neu.ccs.pyramid.elasticsearch.DuplicateDetector;%0Aimport edu.neu.ccs.pyramid.elasticsearch.ESIndex;%0Aimport edu.neu.ccs.pyramid.elasticsearch.SingleLabelIndex;%0A%0Aimport java.io.BufferedWriter;%0Aimport java.io.File;%0Aimport java.io.FileWriter;%0Aimport java.util.regex.Pattern;%0A%0A/**%0A * detect duplicated docs in an index%0A * Created by chengli on 1/20/15.%0A */%0Apublic class Exp56 %7B%0A    public static void main(String%5B%5D args) throws Exception%7B%0A        if (args.length !=1)%7B%0A            throw new IllegalArgumentException(%22please specify the config file%22);%0A        %7D%0A%0A        Config config = new Config(args%5B0%5D);%0A        System.out.println(config);%0A%0A        SingleLabelIndex index = loadIndex(config);%0A        deduplicate(config,index);%0A%0A%0A        index.close();%0A%0A    %7D%0A%0A    static SingleLabelIndex loadIndex(Config config) throws Exception%7B%0A        SingleLabelIndex.Builder builder = new SingleLabelIndex.Builder()%0A                .setIndexName(config.getString(%22index.indexName%22))%0A                .setClusterName(config.getString(%22index.clusterName%22))%0A                .setClientType(config.getString(%22index.clientType%22))%0A                .setLabelField(config.getString(%22index.labelField%22))%0A                .setExtLabelField(config.getString(%22index.extLabelField%22))%0A                .setDocumentType(config.getString(%22index.documentType%22));%0A        if (config.getString(%22index.clientType%22).equals(%22transport%22))%7B%0A            String%5B%5D hosts = config.getString(%22index.hosts%22).split(Pattern.quote(%22,%22));%0A            String%5B%5D ports = config.getString(%22index.ports%22).split(Pattern.quote(%22,%22));%0A            builder.addHostsAndPorts(hosts,ports);%0A        %7D%0A        SingleLabelIndex index = builder.build();%0A        System.out.println(%22index loaded%22);%0A        System.out.println(%22there are %22+index.getNumDocs()+%22 documents in the index.%22);%0A//        for (int i=0;i%3Cindex.getNumDocs();i++)%7B%0A//            System.out.println(i);%0A//            System.out.println(index.getLabel(%22%22+i));%0A//        %7D%0A        return index;%0A    %7D%0A%0A    static void deduplicate(Config config, ESIndex esIndex) throws Exception%7B%0A        DuplicateDetector duplicateDetector = new DuplicateDetector(esIndex);%0A        duplicateDetector.detect();%0A        System.out.println(%22number of duplicates%22);%0A        System.out.println(duplicateDetector.getAllDuplicates().size());%0A        System.out.println(duplicateDetector.toString());%0A        File file = new File(config.getString(%22archive.duplicate%22));%0A        file.getParentFile().mkdirs();%0A        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(file));%0A        bufferedWriter.write(duplicateDetector.toString());%0A        bufferedWriter.close();%0A    %7D%0A%7D%0A

add Exp56