@@ -0,0 +1,1955 @@
+/*%0A * Licensed to the Apache Software Foundation (ASF) under one or more%0A * contributor license agreements.  See the NOTICE file distributed with%0A * this work for additional information regarding copyright ownership.%0A * The ASF licenses this file to You under the Apache License, Version 2.0%0A * (the %22License%22); you may not use this file except in compliance with%0A * the License.  You may obtain a copy of the License at%0A *%0A *   http://www.apache.org/licenses/LICENSE-2.0%0A *%0A * Unless required by applicable law or agreed to in writing, software%0A * distributed under the License is distributed on an %22AS IS%22 BASIS,%0A * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A * See the License for the specific language governing permissions and%0A * limitations under the License.%0A * %0A */%0A%0Apackage org.apache.commons.csv;%0A%0Aimport static org.apache.commons.csv.Token.Type.*;%0A%0A/**%0A * Token is an internal token representation.%0A * %3Cp/%3E%0A * It is used as contract between the lexer and the parser.%0A */%0Aclass Token %7B%0A%0A    /** length of the initial token (content-)buffer */%0A    private static final int INITIAL_TOKEN_LENGTH = 50;%0A    %0A    enum Type %7B%0A        /** Token has no valid content, i.e. is in its initialized state. */%0A        INVALID,%0A        %0A        /** Token with content, at beginning or in the middle of a line. */%0A        TOKEN,%0A        %0A        /** Token (which can have content) when end of file is reached. */%0A        EOF,%0A        %0A        /** Token with content when end of a line is reached. */%0A        EORECORD%0A    %7D%0A    %0A    /** Token type */%0A    Token.Type type = INVALID;%0A    %0A    /** The content buffer. */%0A    StringBuilder content = new StringBuilder(INITIAL_TOKEN_LENGTH);%0A    %0A    /** Token ready flag: indicates a valid token with content (ready for the parser). */%0A    boolean isReady;%0A%0A    Token reset() %7B%0A        content.setLength(0);%0A        type = INVALID;%0A        isReady = false;%0A        return this;%0A    %7D%0A%7D

Move Token into separate file for more flexibility