@@ -0,0 +1,1001 @@
+package kalang.test;%0A%0Aimport java.util.List;%0Aimport kalang.util.TokenStreamFactory;%0Aimport org.antlr.v4.runtime.CommonTokenStream;%0Aimport org.antlr.v4.runtime.IntStream;%0Aimport org.antlr.v4.runtime.Token;%0Aimport org.junit.Test;%0Aimport static org.junit.Assert.*;%0A%0A/**%0A *%0A * @author Kason Yang %3Ci@kasonyang.com%3E%0A */%0Apublic class TokenUtilTest %7B%0A    %0A    public TokenUtilTest() %7B%0A    %7D%0A    %0A    @Test%0A    public void test()%7B%0A        CommonTokenStream ts = TokenStreamFactory.createTokenStream(%22class%7B    %7D%22);%0A        int tokenSize = ts.size();%0A        assertEquals(0, tokenSize);%0A        List%3CToken%3E tokens = ts.getTokens();%0A        assertEquals(0, tokens.size());%0A        ts.consume();%0A        //why is it two?%0A        assertEquals(2, ts.size());%0A        int consumeSize = 1;%0A        while(ts.LA(1)!=IntStream.EOF)%7B%0A            ts.consume();%0A            consumeSize++;%0A        %7D%0A        tokens = ts.getTokens();%0A        assertEquals(5, tokens.size());%0A        assertEquals(3, consumeSize);%0A    %7D%0A    %0A%7D%0A

add test