@@ -0,0 +1,2773 @@
+/* LanguageTool, a natural language style checker%0A * Copyright (C) 2020 Daniel Naber (http://www.danielnaber.de)%0A *%0A * This library is free software; you can redistribute it and/or%0A * modify it under the terms of the GNU Lesser General Public%0A * License as published by the Free Software Foundation; either%0A * version 2.1 of the License, or (at your option) any later version.%0A *%0A * This library is distributed in the hope that it will be useful,%0A * but WITHOUT ANY WARRANTY; without even the implied warranty of%0A * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU%0A * Lesser General Public License for more details.%0A *%0A * You should have received a copy of the GNU Lesser General Public%0A * License along with this library; if not, write to the Free Software%0A * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301%0A * USA%0A */%0Apackage org.languagetool.dev;%0A%0Aimport org.languagetool.DetectedLanguage;%0Aimport org.languagetool.language.LanguageIdentifier;%0A%0Aimport java.io.File;%0Aimport java.io.FileWriter;%0Aimport java.io.IOException;%0Aimport java.util.Collections;%0Aimport java.util.Scanner;%0A%0Apublic class FilterFileByLanguage %7B%0A%0A  private final static String fastTextBinary = %22/prg/fastText-0.1.0/fasttext%22;%0A  private final static String fastTextModel = %22/prg/fastText-0.1.0/data/lid.176.bin%22;%0A  private final static float skipThreshold = 0.98f; // only skip of confidence is higher than this%0A%0A  public static void main(String%5B%5D args) throws IOException %7B%0A    if (args.length != 2) %7B%0A      System.out.println(%22Usage: %22 + FilterFileByLanguage.class.getSimpleName() + %22 %3ClangCode%3E %3Cfile%3E%22);%0A      System.exit(1);%0A    %7D%0A    String expectedLang = args%5B0%5D;%0A    File input = new File(args%5B1%5D);%0A    File output = new File(input.getAbsoluteFile() + %22.filtered%22);%0A    LanguageIdentifier ident = new LanguageIdentifier();%0A    ident.enableFasttext(new File(fastTextBinary), new File(fastTextModel));%0A    Scanner sc = new Scanner(input);%0A    int skipCount = 0;%0A    try (FileWriter fw = new FileWriter(output)) %7B%0A      while (sc.hasNextLine()) %7B%0A        String line = sc.nextLine();%0A        DetectedLanguage lang = ident.detectLanguage(line, Collections.emptyList(), Collections.emptyList());%0A        if (lang != null && !lang.getDetectedLanguage().getShortCode().equals(expectedLang) && lang.getDetectionConfidence() %3E skipThreshold) %7B%0A          System.out.printf(%22Skipping (%25.2f, %25s): %25s%5Cn%22, lang.getDetectionConfidence(), lang.getDetectedLanguage().getShortCode(), line);%0A          skipCount++;%0A        %7D else %7B%0A          fw.write(line);%0A          fw.write(%22%5Cn%22);%0A        %7D%0A      %7D%0A    %7D%0A    System.out.println(skipCount + %22 lines skipped, confidence threshold was %22 + skipThreshold);%0A    System.out.println(%22Filtered result written to %22 + output);%0A  %7D%0A%0A%7D%0A

filter a file (line by line) by a given language