/* LanguageTool, a natural language style checker 
 * Copyright (C) 2014 Daniel Naber (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.bigdata;

import org.languagetool.JLanguageTool;
import org.languagetool.rules.ConfusionSet;
import org.languagetool.rules.ConfusionSetLoader;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;

/**
 * Generate URLs for the confusion set to download the parts of Google's ngram corpus (v2)
 * that we need to cover the confusion set.
 * @since 2.7
 */
final class ConfusionSetUrlGenerator {

  private ConfusionSetUrlGenerator() {}

  public static void main(String[] args) throws IOException {
    ConfusionSetLoader confusionSetLoader =  new ConfusionSetLoader();
    InputStream inputStream = JLanguageTool.getDataBroker().getFromResourceDirAsStream("/en/homophones.txt");
    Map<String,List<ConfusionSet>> map = confusionSetLoader.loadConfusionSet(inputStream);
    String url = "http://storage.googleapis.com/books/ngrams/books/googlebooks-eng-all-2gram-20120701-<XX>.gz";
    Set<String> nameSet = new HashSet<>();
    for (String s : map.keySet()) {
      if (s.length() < 2) {
        nameSet.add(s.substring(0, 1).toLowerCase() + "_");
      } else {
        nameSet.add(s.substring(0, 2).toLowerCase());
      }
    }
    List<String> nameList = new ArrayList<>(nameSet);
    Collections.sort(nameList);
    for (String name : nameList) {
      System.out.println(url.replace("<XX>", name));
    }
    System.err.println("Number of files: " + nameList.size());
  }
}

@@ -1261,16 +1261,78 @@
 2.7%0A */%0A
+@SuppressWarnings(%22DynamicRegexReplaceableByCompiledPattern%22)%0A
 final cl
@@ -1466,16 +1466,730 @@
 ption %7B%0A
+    String url = %22http://storage.googleapis.com/books/ngrams/books/googlebooks-eng-all-4gram-20120701-%3CXX%3E.gz%22;%0A    String chars = %22abcdefghijklmnopqrstuvwxyz%22;%0A    String chars2 = %22abcdefghijklmnopqrstuvwxyz_%22;%0A    for (int i = 0; i %3C= 9; i++) %7B%0A      System.out.println(url.replace(%22%3CXX%3E%22, String.valueOf(i)));%0A    %7D%0A    for (int i = 0; i %3C chars.length(); i++) %7B%0A      for (int j = 0; j %3C chars2.length(); j++) %7B%0A        String name = String.valueOf(chars.charAt(i)) + String.valueOf(chars2.charAt(j));%0A        System.out.println(url.replace(%22%3CXX%3E%22, name));%0A      %7D%0A    %7D%0A    System.out.println(url.replace(%22%3CXX%3E%22, %22punctuation%22));%0A  %7D%0A%0A  public static void mainDownloadSome(String%5B%5D args) throws IOException %7B%0A
     Conf

extend to cover all urls