@@ -0,0 +1,2509 @@
+package focusedCrawler.tools;%0A%0Aimport java.io.PrintStream;%0Aimport java.util.HashMap;%0Aimport java.util.Map;%0A%0Aimport com.fasterxml.jackson.databind.ObjectMapper;%0Aimport com.google.common.hash.Hashing;%0A%0Aimport focusedCrawler.target.model.TargetModelJson;%0Aimport focusedCrawler.target.repository.FilesTargetRepository;%0Aimport focusedCrawler.target.repository.FilesTargetRepository.RepositoryIterator;%0Aimport focusedCrawler.util.CliTool;%0Aimport io.airlift.airline.Command;%0Aimport io.airlift.airline.Option;%0A%0A@Command(name=%22DumpMetadataFromRepository%22)%0Apublic class DumpMetadataFromRepository extends CliTool %7B%0A%0A    @Option(name = %22--input-path%22,%0A            required = true,%0A            description = %22Path to directory containing a FILES repository%22)%0A    private String inputPath;%0A%0A    @Option(name = %22--output-file%22,%0A            required = true,%0A            description = %22Text file containing metadata%22)%0A    private String outputFile;%0A%0A    public static void main(String%5B%5D args) throws Exception %7B%0A        CliTool.run(args, new DumpMetadataFromRepository());%0A    %7D%0A%0A    @Override%0A    public void execute() throws Exception %7B%0A%0A        System.out.println(%22Reading URLs from path: %22 + inputPath);%0A        %0A        FilesTargetRepository repository = new FilesTargetRepository(inputPath);%0A        ObjectMapper jsonMapper = new ObjectMapper();%0A        %0A        PrintStream out = System.out;%0A        if(outputFile != null) %7B%0A            out  = new PrintStream(outputFile);%0A        %7D%0A        %0A        int totalPages = 0;%0A        RepositoryIterator it = repository.iterator();%0A        while(it.hasNext()) %7B%0A            TargetModelJson page = it.next();%0A%0A            String id = Hashing.sha256().hashBytes(page.getUrl().getBytes()).toString();%0A            String signature = Hashing.md5().hashBytes(page.getContent()).toString();%0A            %0A            Map%3CString, Object%3E obj = new HashMap%3CString, Object%3E();%0A            obj.put(%22id%22, id);%0A            obj.put(%22url%22, page.getUrl());%0A            obj.put(%22signature%22, signature);%0A            obj.put(%22score%22, page.getRelevance().getRelevance());%0A            obj.put(%22team%22, %22NYU%22);%0A            obj.put(%22timestamp_fetch%22, page.getFetchTime());%0A            %0A            out.println(jsonMapper.writeValueAsString(obj));%0A            %0A            totalPages++;%0A            if (totalPages %25 1000 == 0) %7B%0A                System.out.printf(%22Processed %25s pages...%5Cn%22, totalPages);%0A            %7D%0A        %7D%0A        repository.close();%0A        System.out.println(%22done.%22);%0A    %7D%0A%0A%7D%0A

Add tool for dumping some metadata from repository