package org.duyi;

import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
import org.json.JSONObject;

import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

/**
 * Created by yidu on 11/10/15.
 * read file and store weather data to mongodb
 */
public class WeatherDataProcessor {
    public static void main(String[] args){
        try {
            MongoClient client = new MongoClient("127.0.0.1");
            MongoDatabase db = client.getDatabase("pm");
            MongoCollection weather = db.getCollection("weatherRaw");
            Document doc;

            File dir = new File("/Users/yidu/Downloads/airData/");
            File[] files = dir.listFiles();
            for (File f : files) {
                if(!f.getName().startsWith("S2015"))
                    continue;
                System.out.println(f.getName());
                List<String> lines = IOUtils.readLines(new FileInputStream(f));
                StringTokenizer st = new StringTokenizer(lines.get(0));
                //get header
                ArrayList<String> header = new ArrayList<String>();
                while(st.hasMoreTokens()){
                    header.add(st.nextToken());
                }

                for(int i = 1; i < lines.size(); i ++){
                    JSONObject obj = new JSONObject();
                    StringTokenizer st2 = new StringTokenizer(lines.get(i));
                    doc = new Document();
                    for(int j = 0; j < header.size(); j ++){
                        doc.append(header.get(j), st2.nextToken());
                    }
                    weather.insertOne(doc);
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }

    }
}

@@ -87,16 +87,55 @@
 ection;%0A
+import com.mongodb.client.MongoCursor;%0A
 import c
@@ -313,16 +313,51 @@
 Stream;%0A
+import java.text.SimpleDateFormat;%0A
 import j
@@ -476,135 +476,1690 @@
 .%0A *
- read file and store weather data to mongodb%0A */%0Apublic class WeatherDataProcessor %7B%0A    public static void main(String%5B%5D args)
+%0A */%0Apublic class WeatherDataProcessor %7B%0A    static SimpleDateFormat df = new SimpleDateFormat(%22yyyy-MM-dd Z%22);%0A    static ArrayList%3CString%3E header;%0A    static MongoClient client;%0A    static MongoDatabase db;%0A%0A    public static void main(String%5B%5D args)%7B%0A        client = new MongoClient(%22127.0.0.1%22);%0A        db = client.getDatabase(%22pm%22);%0A        loadAndStore();%0A        preProcess();%0A    %7D%0A%0A    /**%0A     * parse date and other digital%0A     */%0A    private static void preProcess() %7B%0A        try %7B%0A            MongoCollection weatherRaw = db.getCollection(%22weatherRaw%22);%0A            MongoCollection weather = db.getCollection(%22weather%22);%0A            MongoCursor cursor = weatherRaw.find().iterator();%0A            while (cursor.hasNext()) %7B%0A                Document insert = new Document();%0A                Document info = (Document) cursor.next();%0A                String year = info.getString(%22Year%22);%0A                String month = info.getString(%22Mon%22);%0A                String day = info.getString(%22Day%22);%0A                insert.append(%22station%22, Integer.parseInt(info.getString(%22Station_Id_C%22)));%0A                insert.append(%22time%22, df.parse(year + %22-%22 + month + %22-%22 + day + %22 +0000%22));%0A                for(String h : header)%7B%0A                    if(h.equals(%22Year%22) %7C%7C h.equals(%22Mon%22) %7C%7C h.equals(%22Day%22))%0A                        continue;%0A                    insert.append(h, Double.parseDouble(info.getString(h)));%0A                %7D%0A                weather.insertOne(insert);%0A            %7D%0A        %7Dcatch(Exception e)%7B%0A            e.printStackTrace();%0A        %7D%0A    %7D%0A%0A    /**%0A     * read file and store weather data to mongodb%0A     */%0A    private static void loadAndStore() 
 %7B%0A  
@@ -2861,34 +2861,16 @@
         
- ArrayList%3CString%3E
  header 
@@ -3535,17 +3535,18 @@
       %7D%0A
-%0A
     %7D%0A
+%0A%0A
 %7D%0A

add preprocessor of weather dataset