/* LanguageTool, a natural language style checker
 * Copyright (C) 2013 Daniel Naber (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.dumpcheck;

import org.languagetool.Language;

import java.io.InputStream;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Provides access to the sentences of a Tatoeba (http://tatoeba.org) text
 * file (tab separated) that has already been filtered to contain only one language.
 * @since 2.4
 */
class TatoebaSentenceSource extends SentenceSource {

  private final List<String> sentences;
  private final Scanner scanner;

  // Each sentence is one article, but count anyway so it's coherent with what the Wikipedia code does:
  private int articleCount = 0;
  
  TatoebaSentenceSource(InputStream textInput, Language language) {
    this(textInput, language, null);
  }

  /** @since 3.0 */
  TatoebaSentenceSource(InputStream textInput, Language language, Pattern filter) {
    super(language, filter);
    scanner = new Scanner(textInput);
    sentences = new ArrayList<>();
  }

  @Override
  public boolean hasNext() {
    fillSentences();
    return sentences.size() > 0;
  }

  @Override
  public Sentence next() {
    fillSentences();
    if (sentences.isEmpty()) {
      throw new NoSuchElementException();
    }
    return new Sentence(sentences.remove(0), getSource(), "<Tatoeba>", "http://tatoeba.org", ++articleCount);
  }

  @Override
  public String getSource() {
    return "tatoeba";
  }

  private void fillSentences() {
    while (sentences.isEmpty() && scanner.hasNextLine()) {
      String line = scanner.nextLine();
      if (line.isEmpty()) {
        continue;
      }
      String[] parts = line.split("\t");
      if (parts.length != 3) {
        System.err.println("Unexpected line format: expected three tab-separated columns: '" + line  + "', skipping");
        continue;
      }
      String sentence = parts[2];  // actually it's sometimes two (short) sentences, but anyway...
      if (acceptSentence(sentence)) {
        sentences.add(sentence);
      }
    }
  }

}

@@ -1266,22 +1266,31 @@
 al List%3C
-String
+TatoebaSentence
 %3E senten
@@ -2023,24 +2023,121 @@
 on();%0A    %7D%0A
+    TatoebaSentence sentence = sentences.remove(0);%0A    String title = %22Tatoeba-%22 + sentence.id;%0A
     return n
@@ -2156,27 +2156,25 @@
 sentence
-s.remove(0)
+.sentence
 , getSou
@@ -2184,19 +2184,13 @@
 (), 
-%22%3CTatoeba%3E%22
+title
 , %22h
@@ -2695,32 +2695,74 @@
 ntinue;%0A      %7D%0A
+      long id = Long.parseLong(parts%5B0%5D);%0A
       String sen
@@ -2904,16 +2904,40 @@
 ces.add(
+new TatoebaSentence(id, 
 sentence
@@ -2937,16 +2937,17 @@
 entence)
+)
 ;%0A      
@@ -2959,10 +2959,227 @@
  %7D%0A  %7D%0A%0A
+  private static class TatoebaSentence %7B%0A    long id;%0A    String sentence;%0A    private TatoebaSentence(long id, String sentence) %7B%0A      this.id = id;%0A      this.sentence = Objects.requireNonNull(sentence);%0A    %7D%0A  %7D%0A
 %7D%0A

add id to Tatoeba sentence - will later help keeping matches apart