/* LanguageTool, a natural language style checker
 * Copyright (C) 2020 Daniel Naber (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev;

import org.languagetool.AnalyzedTokenReadings;
import org.languagetool.languagemodel.LuceneLanguageModel;
import org.languagetool.tagging.de.GermanTagger;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;

/**
 * Fast hack to find words which have no POS tagging.
 */
public class MissingGermanPosFinder {

  public static void main(String[] args) throws IOException {
    if (args.length != 2) {
      System.out.println("Usage: " + MissingGermanPosFinder.class.getSimpleName() + " <file> <ngram_dir>");
      System.exit(1);
    }
    List<String> lines = Files.readAllLines(Paths.get(args[0]));
    //List<String> lines = Arrays.asList("Bundesrepublik", "Landwirtschaft", "Perl", "Haus", "Drücke", "Wischdsda", "gut", "schönxxx");
    LuceneLanguageModel lm = new LuceneLanguageModel(new File(args[1]));
    GermanTagger tagger = new GermanTagger();
    for (String word : lines) {
      word = word.trim();
      AnalyzedTokenReadings matches = tagger.lookup(word);
      if (matches == null || !matches.isTagged()) {
        long count = lm.getCount(word);
        System.out.println(count + "\t" + word);
      }
    }
  }
}

@@ -1857,84 +1857,436 @@
     
-word = word.trim();%0A      AnalyzedTokenReadings matches = tagger.lookup(word
+int origCount = -1;%0A      if (word.matches(%22%5C%5Cd+ .*%22)) %7B%0A        String%5B%5D parts = word.split(%22 %22);%0A        origCount = Integer.parseInt(parts%5B0%5D);%0A        word = parts%5B1%5D;%0A      %7D%0A      word = word.trim();%0A      if (word.endsWith(%22.%22)) %7B%0A        word = word.substring(0, word.length()-1);%0A      %7D%0A      AnalyzedTokenReadings matches = tagger.lookup(word);%0A      AnalyzedTokenReadings lcMatches = tagger.lookup(word.toLowerCase()
 );%0A 
@@ -2293,16 +2293,17 @@
      if 
+(
 (matches
@@ -2334,16 +2334,65 @@
 agged())
+ && (lcMatches == null %7C%7C !lcMatches.isTagged()))
  %7B%0A     
@@ -2406,16 +2406,34 @@
  count =
+ origCount == -1 ?
  lm.getC
@@ -2438,24 +2438,36 @@
 tCount(word)
+ : origCount
 ;%0A        Sy

deal with optional numbers in each line