@@ -0,0 +1,1719 @@
+package org.nwapw.abacus.tests;%0A%0Aimport org.junit.Assert;%0Aimport org.junit.Test;%0Aimport org.nwapw.abacus.lexing.pattern.Match;%0Aimport org.nwapw.abacus.parsing.LexerTokenizer;%0Aimport org.nwapw.abacus.tree.TokenType;%0A%0Aimport java.util.List;%0A%0Apublic class TokenizerTests %7B%0A%0A    private LexerTokenizer lexerTokenizer = new LexerTokenizer();%0A%0A    @Test%0A    public void testInteger()%7B%0A        List%3CMatch%3CTokenType%3E%3E tokens = lexerTokenizer.tokenizeString(%2211%22);%0A        Assert.assertNotNull(tokens);%0A        Assert.assertEquals(tokens.size(), 1);%0A        Assert.assertEquals(tokens.get(0).getType(), TokenType.NUM);%0A    %7D%0A%0A    @Test%0A    public void testLeadingZeroDecimal()%7B%0A        List%3CMatch%3CTokenType%3E%3E tokens = lexerTokenizer.tokenizeString(%220.1%22);%0A        Assert.assertNotNull(tokens);%0A        Assert.assertEquals(tokens.size(), 1);%0A        Assert.assertEquals(tokens.get(0).getType(), TokenType.NUM);%0A    %7D%0A%0A    @Test%0A    public void testNonLeadingDecimal()%7B%0A        List%3CMatch%3CTokenType%3E%3E tokens = lexerTokenizer.tokenizeString(%22.1%22);%0A        Assert.assertNotNull(tokens);%0A        Assert.assertEquals(tokens.size(), 1);%0A        Assert.assertEquals(tokens.get(0).getType(), TokenType.NUM);%0A    %7D%0A%0A    @Test%0A    public void testSimpleChars()%7B%0A        List%3CMatch%3CTokenType%3E%3E tokens = lexerTokenizer.tokenizeString(%22( ,)%22);%0A        Assert.assertNotNull(tokens);%0A        Assert.assertEquals(tokens.size(), 4);%0A        Assert.assertEquals(tokens.get(0).getType(), TokenType.OPEN_PARENTH);%0A        Assert.assertEquals(tokens.get(1).getType(), TokenType.WHITESPACE);%0A        Assert.assertEquals(tokens.get(2).getType(), TokenType.COMMA);%0A        Assert.assertEquals(tokens.get(3).getType(), TokenType.CLOSE_PARENTH);%0A    %7D%0A%0A%7D%0A

Write basic tokenizer tests.