package boa.datagen.forges.github;

import java.io.File;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;

import boa.datagen.util.FileIO;
import gnu.trove.set.hash.THashSet;

public class DuplicateRepoWorker implements Runnable {
	private final String inPath;
	private final String outPath;
	private final int to;
	private final int from;
	THashSet<String> names = DuplicateRepoRemover.names;
	private JsonArray cleared = new JsonArray();
	private int counter = 0;
	
	
	public DuplicateRepoWorker(String inPath, String outPath, int from, int to ){
		this.inPath = inPath;
		this.outPath = outPath;
		this.to = to;
		this.from = from;
	}
	
	private void removeDuplicates(){
		System.out.println(Thread.currentThread().getId() + " Responsible for processing: " + from + " and " + to);
		File[] files = new File(inPath).listFiles(); 
		for (int i = from; i < to; i++){
			File repoFile = files[i];
			String content = FileIO.readFileContents(repoFile);
			Gson parser = new Gson();
			JsonArray repos = parser.fromJson(content, JsonElement.class).getAsJsonArray();
			int size = repos.size();
			for (int j = 0; j < size; j++) {
				JsonObject repo = repos.get(j).getAsJsonObject();
				String name = repo.get("full_name").getAsString();
				if (names.contains(name)) {
					System.out.println("found a duplicate " + name);
					continue;
				}
				names.add(name);
				int id = repo.get("id").getAsInt();
				JsonObject owner = repo.get("owner").getAsJsonObject();
				String created_at = null;
				if (repo.has("created_at"))
					created_at = repo.get("created_at").getAsString();
				boolean fork = repo.get("fork").getAsBoolean();
				String description = null;
				if (!repo.get("description").isJsonNull())
					description = repo.get("description").getAsString();
				JsonObject langList = new JsonObject();
				if (repo.has("language_list")){
					if(!repo.get("language_list").isJsonObject()){
						String brokenList = repo.get("language_list").getAsString();
						brokenList = brokenList.replace("\\", "");
						repo.remove("language_list");
						repo.addProperty("language_list", brokenList);
					}
					if (repo.get("language_list").isJsonObject())
						langList = repo.get("language_list").getAsJsonObject();
				}
				
				JsonObject reducedRepo = new JsonObject();
				reducedRepo.addProperty("id", id);
				reducedRepo.addProperty("full_name", name);
				reducedRepo.add("owner", owner);
				reducedRepo.addProperty("created_at", created_at);
				reducedRepo.addProperty("fork", fork);
				reducedRepo.addProperty("description", description);
				reducedRepo.add("language_list", langList);
				this.addRepo(reducedRepo);
			}
			System.out.println(Thread.currentThread().getId() + " processing: " + i);
		}
		this.writeRemainingRepos();
		System.out.print(Thread.currentThread().getId() + "finished");
	}
	
	private void writeRemainingRepos() {
		if (cleared.size() > 0)
			FileIO.writeFileContents(new File(outPath + "/" + Thread.currentThread().getId() + "-page" + counter + ".json"), cleared.toString());
	}

	private void addRepo(JsonObject repo) {
		cleared.add(repo);
		if (cleared.size() >= 100){
			FileIO.writeFileContents(new File(outPath + "/" + Thread.currentThread().getId() + "-page" + counter + ".json"), cleared.toString());
			cleared = new JsonArray();
			counter ++;
		}
	}

	@Override
	public void run() {
		this.removeDuplicates();
	}

}

@@ -2971,24 +2971,25 @@
 e() %3E 0)
+%7B
 %0A%09%09%09File
 IO.write
@@ -2980,37 +2980,35 @@
 %0A%09%09%09File
-IO.writeFileContents(
+ fileToWriteJson = 
 new File
@@ -3085,38 +3085,302 @@
 %22.json%22)
-, clea
+;%0A%09%09%09while (fileToWriteJson.exists()) %7B%0A%09%09%09%09System.out.println(%22file thread-%22 + Th
 re
+a
 d.
-toString());
+currentThread().getId() + %22-page-%22 + counter + %22 arleady exist%22);%0A%09%09%09%09counter++;%0A%09%09%09%09fileToWriteJson = new File(outPath + %22/%22 + Thread.currentThread().getId() + %22-page%22 + counter + %22.json%22);%0A%09%09%09%7D%0A%09%09%7D
 %0A%09%7D%0A%0A%09pr
@@ -3479,29 +3479,27 @@
 File
-IO.writeFileContents(
+ fileToWriteJson = 
 new 
@@ -3584,74 +3584,319 @@
 on%22)
-, cleared.toString());%0A%09%09%09cleared = new JsonArray();%0A%09%09%09counter ++
+;%0A%09%09%09while (fileToWriteJson.exists()) %7B%0A%09%09%09%09System.out.println(%22file thread-%22 + Thread.currentThread().getId() + %22-page-%22 + counter + %22 arleady exist%22);%0A%09%09%09%09counter++;%0A%09%09%09%09fileToWriteJson = new File(outPath + %22/%22 + Thread.currentThread().getId() + %22-page%22 + counter + %22.json%22);%0A%09%09%09%7D%0A%09%09%09cleared = new JsonArray()
 ;%0A%09%09

check if fileToWriteJson exists