@@ -0,0 +1,3133 @@
+/*%0A * Licensed to the Apache Software Foundation (ASF) under one or more%0A * contributor license agreements.  See the NOTICE file distributed with%0A * this work for additional information regarding copyright ownership.%0A * The ASF licenses this file to You under the Apache License, Version 2.0%0A * (the %22License%22); you may not use this file except in compliance with%0A * the License. You may obtain a copy of the License at%0A *%0A *     http://www.apache.org/licenses/LICENSE-2.0%0A *%0A * Unless required by applicable law or agreed to in writing, software%0A * distributed under the License is distributed on an %22AS IS%22 BASIS,%0A * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A * See the License for the specific language governing permissions and%0A * limitations under the License.%0A */%0A%0Apackage opennlp.tools.tokenize;%0A%0Aimport java.io.IOException;%0A%0Aimport org.junit.Assert;%0Aimport org.junit.Test;%0A%0Aimport opennlp.tools.util.ObjectStream;%0Aimport opennlp.tools.util.ObjectStreamUtils;%0Aimport opennlp.tools.util.Span;%0A%0Apublic class WhitespaceTokenStreamTest %7B%0A%0A  /**%0A   * Tests for the %7B@link WhitespaceTokenStream%7D class.%0A   */%0A  @Test%0A  public void testWhitespace() throws IOException %7B%0A    String text = %22 a b c  d    e        f     %22;%0A    ObjectStream%3CTokenSample%3E sampleStream = new TokenSampleStream(%0A            ObjectStreamUtils.createObjectStream(text));%0A    WhitespaceTokenStream tokenStream = new WhitespaceTokenStream(sampleStream);%0A    String read = tokenStream.read();%0A    Assert.assertEquals(%22a b c d e f%22, read);%0A  %7D%0A%0A  @Test%0A  public void testSeparatedString() throws IOException %7B%0A    String text = %22 a b%3CSPLIT%3Ec   d%3CSPLIT%3Ee   %22;%0A    ObjectStream%3CTokenSample%3E sampleStream = new TokenSampleStream(%0A            ObjectStreamUtils.createObjectStream(text));%0A    WhitespaceTokenStream tokenStream = new WhitespaceTokenStream(sampleStream);%0A    String read = tokenStream.read();%0A    Assert.assertEquals(%22a b c d e%22, read);%0A  %7D%0A%0A  /**%0A   * Tests for the %7B@link TokenizerStream%7D correctly tokenizes whitespace separated tokens.%0A   */%0A  @Test%0A  public void testTokenizerStream() throws IOException %7B%0A    String text = %22 a b c  d    e      %22;%0A    WhitespaceTokenizer instance = WhitespaceTokenizer.INSTANCE;%0A    TokenizerStream stream = new TokenizerStream(instance, ObjectStreamUtils.createObjectStream(text));%0A    TokenSample read = stream.read();%0A    Span%5B%5D tokenSpans = read.getTokenSpans();%0A%0A    Assert.assertEquals(5, tokenSpans.length);%0A%0A    Assert.assertEquals(%22a%22, tokenSpans%5B0%5D.getCoveredText(read.getText()));%0A    Assert.assertEquals(new Span(1,2), tokenSpans%5B0%5D);%0A%0A    Assert.assertEquals(%22b%22, tokenSpans%5B1%5D.getCoveredText(read.getText()));%0A    Assert.assertEquals(new Span(3,4), tokenSpans%5B1%5D);%0A%0A    Assert.assertEquals(%22c%22, tokenSpans%5B2%5D.getCoveredText(read.getText()));%0A    Assert.assertEquals(new Span(5,6), tokenSpans%5B2%5D);%0A%0A    Assert.assertEquals(%22d%22, tokenSpans%5B3%5D.getCoveredText(read.getText()));%0A    Assert.assertEquals(new Span(8,9), tokenSpans%5B3%5D);%0A%0A    Assert.assertEquals(%22e%22, tokenSpans%5B4%5D.getCoveredText(read.getText()));%0A    Assert.assertEquals(new Span(13,14), tokenSpans%5B4%5D);%0A  %7D%0A%7D%0A

add WhitespaceTokenStreamTest (#379)