package io.cucumber.cucumberexpressions;

import io.cucumber.cucumberexpressions.Ast.Token;
import io.cucumber.cucumberexpressions.Ast.Token.Type;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.PrimitiveIterator.OfInt;

import static io.cucumber.cucumberexpressions.CucumberExpressionException.createCantEscape;
import static io.cucumber.cucumberexpressions.CucumberExpressionException.createTheEndOfLineCanNotBeEscapedException;

final class CucumberExpressionTokenizer {

    List<Token> tokenize(String expression) {
        List<Token> tokens = new ArrayList<>();
        tokenizeImpl(expression).forEach(tokens::add);
        return tokens;
    }

    private Iterable<Token> tokenizeImpl(String expression) {
        return () -> new TokenIterator(expression);

    }

    private static class TokenIterator implements Iterator<Token> {
        final OfInt codePoints;
        private final String expression;
        StringBuilder buffer;
        Type previousTokenType;
        Type currentTokenType;
        boolean treatAsText;
        int index;
        int escaped;

        TokenIterator(String expression) {
            this.expression = expression;
            codePoints = expression.codePoints().iterator();
            buffer = new StringBuilder();
            previousTokenType = null;
            currentTokenType = Type.START_OF_LINE;
            treatAsText = false;
            index = 0;
            escaped = 0;
        }

        @Override
        public boolean hasNext() {
            return previousTokenType != Type.END_OF_LINE;
        }

        @Override
        public Token next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            if (currentTokenType == Type.START_OF_LINE) {
                Token token = convertBufferToToken(currentTokenType);
                advanceTokenTypes();
                return token;
            }

            while (codePoints.hasNext()) {
                int token = codePoints.nextInt();
                if (!treatAsText && Token.isEscapeCharacter(token)) {
                    escaped++;
                    treatAsText = true;
                    continue;
                }
                currentTokenType = tokenTypeOf(token, treatAsText);
                treatAsText = false;

                if (previousTokenType != Type.START_OF_LINE
                        && (currentTokenType != previousTokenType
                        || (currentTokenType != Type.WHITE_SPACE && currentTokenType != Type.TEXT))) {
                    Token t = convertBufferToToken(previousTokenType);
                    advanceTokenTypes();
                    buffer.appendCodePoint(token);
                    return t;
                } else {
                    advanceTokenTypes();
                    buffer.appendCodePoint(token);
                }
            }

            if (buffer.length() > 0) {
                Token token = convertBufferToToken(previousTokenType);
                advanceTokenTypes();
                return token;
            }

            currentTokenType = Type.END_OF_LINE;
            if (treatAsText) {
                throw createTheEndOfLineCanNotBeEscapedException(expression);
            }
            Token token = convertBufferToToken(currentTokenType);
            advanceTokenTypes();
            return token;
        }

        private void advanceTokenTypes() {
            previousTokenType = currentTokenType;
            currentTokenType = null;
        }

        private Token convertBufferToToken(Type currentTokenType) {
            int escapeTokens = 0;
            if (currentTokenType == Type.TEXT) {
                escapeTokens = escaped;
                escaped = 0;
            }
            int endIndex = index + buffer.codePointCount(0, buffer.length()) + escapeTokens;
            Token t = new Token(buffer.toString(), currentTokenType, index, endIndex);
            buffer = new StringBuilder();
            this.index = endIndex;
            return t;
        }

        private Type tokenTypeOf(Integer token, boolean treatAsText) {
            if (!treatAsText) {
                return Token.typeOf(token);
            }
            if (Token.canEscape(token)) {
                return Type.TEXT;
            }
            throw createCantEscape(expression, index + escaped);
        }

    }

}

@@ -930,39 +930,8 @@
 %3E %7B%0A
-        final OfInt codePoints;
 %0A   
@@ -980,28 +980,99 @@
     
-StringBuilder buffer
+private final OfInt codePoints;%0A%0A        private StringBuilder buffer = new StringBuilder()
 ;%0A  
@@ -1073,24 +1073,32 @@
 ();%0A        
+private 
 Type previou
@@ -1117,16 +1117,24 @@
         
+private 
 Type cur
@@ -1138,32 +1138,53 @@
 currentTokenType
+ = Type.START_OF_LINE
 ;%0A        boolea
@@ -1177,16 +1177,24 @@
         
+private 
 boolean 
@@ -1210,24 +1210,32 @@
 xt;%0A        
+private 
 int index;%0A 
@@ -1233,32 +1233,40 @@
  index;%0A        
+private 
 int escaped;%0A%0A  
@@ -1360,16 +1360,21 @@
         
+this.
 codePoin
@@ -1418,220 +1418,8 @@
 ();%0A
-            buffer = new StringBuilder();%0A            previousTokenType = null;%0A            currentTokenType = Type.START_OF_LINE;%0A            treatAsText = false;%0A            index = 0;%0A            escaped = 0;%0A
     

Structure and naming