/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.tokenize;

import opennlp.tools.util.Span;

/**
 * The interface for tokenizers, which segment a string into its token and
 * usually to segment a text into its constituent tokens.
 * 
 * Tokenization is a necessary step before more complex NLP task can be applied,
 * these usually process text on a token level. The quality of tokenization is
 * important because it influences the performance of high-level task applied to it.
 * 
 * In segmented languages like English most words are segmented by white spaces
 * expect for punctuations, etc. which is directly attached to the word without a white space
 * in between, but its not possible to just split all punctuations because in abbrevation dots
 * are a part of the token itself. A tokenizer is now responsible to split these tokens
 * correctly.
 * 
 * In non-segmented languages like Chinese tokenization is more difficult since words
 * are not segmented by a whitespace.
 *  
 * Tokenizers can also be used to segment already identified tokens further into more
 * atomic parts to get a deeper understanding. This approach helps more complex task
 * to gain insight into tokens which do not represent words like numbers, units or tokens
 * which are part of a special notation.
 * 
 * For most further task it is desirable to over tokenize rather than under tokenize.
 */
public interface Tokenizer {

    /**
     * Splits a string into its atomic parts
     *
     * @param s The string to be tokenized.
     * @return  The String[] with the individual tokens as the array
     *          elements.
     */
    public String[] tokenize(String s);

    /**
     * Finds the boundaries of atomic parts in a string.
     *
     * @param s The string to be tokenized.
     * @return The Span[] with the spans (offsets into s) for each
     * token as the individuals array elements.
     */
    public Span[] tokenizePos(String s);
}

@@ -935,68 +935,8 @@
 its 
-token and%0A * usually to segment a text into its constituent 
 toke
@@ -1007,16 +1007,17 @@
 NLP task
+s
  can be 
@@ -1386,14 +1386,12 @@
 en, 
-bu
+i
 t i
-t
 s no
@@ -1415,16 +1415,19 @@
 t split 
+at 
 all punc
@@ -1452,21 +1452,23 @@
 n abbrev
+i
 ation
+s
  dots%0A *

Corrected mistakes