package edu.neu.ccs.pyramid.experiment;

import edu.neu.ccs.pyramid.configuration.Config;
import edu.neu.ccs.pyramid.data_formatter.ohsumed.IndexBuilder;

import edu.neu.ccs.pyramid.util.DirWalker;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.logging.log4j.Log4jESLoggerFactory;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.node.Node;

import java.io.File;
import java.io.FileReader;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.node.NodeBuilder.nodeBuilder;

/**
 * index Ohsumed_20000
 * Created by chengli on 10/1/14.
 */
public class Exp8 {
    public static void main(String[] args) throws Exception{

        if (args.length !=1){
            throw new IllegalArgumentException("please specify the config file");
        }

        Config config = new Config(args[0]);
        System.out.println(config);

        String dir  = config.getString("folder");

        Node node = nodeBuilder().client(true).clusterName(config.getString("index.name")).node();
        Client client = node.client();

        List<File> list = DirWalker.getFiles(dir);
        Map<String, Set<String>> nameToCodesMap = IndexBuilder.collectCodes(dir);
        int id = 0;
        for (File file: list){
            System.out.println("id = "+id);
            XContentBuilder builder = IndexBuilder.getBuilder(file,nameToCodesMap);
            System.out.println(builder.string());
//            IndexResponse response = client.prepareIndex("ohsumed_20000", "document",""+id)
//                    .setSource(builder)
//                    .execute()
//                    .actionGet();
            id += 1;
        }
        node.close();
    }


}

@@ -548,24 +548,50 @@
 FileReader;%0A
+import java.util.HashSet;%0A
 import java.
@@ -601,16 +601,16 @@
 l.List;%0A
-
 import j
@@ -1294,32 +1294,129 @@
 .getFiles(dir);%0A
+        //make sure each file is indexed only once%0A        Set%3CString%3E added = new HashSet%3C%3E();%0A%0A
         Map%3CStri
@@ -1532,16 +1532,70 @@
  list)%7B%0A
+            if (!added.contains(file.getName()))%7B%0A    
         
@@ -1630,16 +1630,20 @@
  %22+id);%0A
+    
         
@@ -1718,16 +1718,20 @@
 esMap);%0A
+//  
         
@@ -1726,32 +1726,33 @@
 //              
+ 
 System.out.print
@@ -1773,18 +1773,20 @@
 ing());%0A
-//
+    
         
@@ -1869,18 +1869,20 @@
 ,%22%22+id)%0A
-//
+    
         
@@ -1913,18 +1913,20 @@
 uilder)%0A
-//
+    
         
@@ -1948,18 +1948,20 @@
 ecute()%0A
-//
+    
         
@@ -1998,16 +1998,20 @@
         
+    
 id += 1;
@@ -2011,16 +2011,171 @@
 d += 1;%0A
+                added.add(file.getName());%0A            %7D else %7B%0A                System.out.println(file.getName()+%22 already indexed, skip%22);%0A            %7D%0A
         

avoid duplicates