package org.summarly.lib.segmentation;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ru.RussianAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import org.summarly.lib.common.Text;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 */
public class LuceneSplitter implements TextSplitter {

    public List<String> splitSentence(String text) throws IOException {
        final Analyzer analyzer;

        analyzer = new RussianAnalyzer(Version.LUCENE_4_9);

        return extractNormalizedTokens(text, analyzer);

    }

    private List<String> extractNormalizedTokens(String text, Analyzer analyzer) throws IOException {
        try (TokenStream tokenStream = analyzer.tokenStream(null, text)) {
            tokenStream.reset();
            ArrayList<String> words = new ArrayList<>();
            while (tokenStream.incrementToken()) {
                words.add(tokenStream.getAttribute(CharTermAttribute.class).toString());
            }
            return words;
        }
    }

    @Override
    public Text split(String text, String name) {

        return null;
    }
}

@@ -282,24 +282,65 @@
 il.Version;%0A
+import org.summarly.lib.common.Sentence;%0A
 import org.s
@@ -819,35 +819,16 @@
 nalyzer)
- throws IOException
  %7B%0A     
@@ -1167,16 +1167,93 @@
  words;%0A
+        %7D catch (IOException e) %7B%0A            throw new RuntimeException(e);%0A
         
@@ -1329,28 +1329,417 @@
 ) %7B%0A
-%0A        return null
+        String sentence = text; //TODO separate sentences%0A%0A        ArrayList%3CSentence%3E sentences = new ArrayList%3C%3E();%0A        Sentence aSentence = new Sentence(sentence);%0A        aSentence.setWords(extractNormalizedTokens(text, new RussianAnalyzer(Version.LUCENE_4_9)));%0A        sentences.add(aSentence);%0A%0A        Text theText = new Text(text);%0A        theText.setSentences(sentences);%0A%0A        return theText
 ;%0A  

Implementation of Russian sentence tokenizer