package com.github.kmizu.jcombinator.tokenizer;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.github.kmizu.jcombinator.core.Functions.*;
import static com.github.kmizu.jcombinator.core.Tuples.*;

public class TokenizerSpecification<T> {
    private final List<Tp2<Pattern, Fn1<String, T>>> specs = new ArrayList<>();
    private Pattern skipPattern = Pattern.compile("");
    private final String input;

    public TokenizerSpecification(String input) {
        this.input = input;
    }

    public final void register(Pattern pattern, Fn1<String, T> action) {
        specs.add(new Tp2<>(pattern, action));
    }

    public final void register(String pattern, Fn1<String, T> action) {
        register(Pattern.compile(pattern), action);
    }

    public final void registerSkipPattern(Pattern skipPattern) {
        this.skipPattern = skipPattern;
    }

    public final void registerSkipPattern(String skipPattern) {
        this.skipPattern = Pattern.compile(skipPattern);
    }

    public final Tokenizer tokenizer() {
        return new Tokenizer<T>(){
            private String next = input;
            private T currentToken;
            @Override
            public boolean moveNext() {
                for(Tp2<Pattern, Fn1<String, T>> spec:specs) {
                    boolean shouldHaveNext = spec.extract((pattern, action) -> {
                        Matcher matcher = pattern.matcher(next);
                        if(matcher.find(0)) {
                            currentToken = action.invoke(next.substring(matcher.start(), matcher.end()));
                            return true;
                        } else {
                            currentToken = null;
                            return false;
                        }
                    });
                    if(shouldHaveNext) return true;
                }
                return false;
            }

            @Override
            public T current() {
                return currentToken;
            }
        };
    }
}

@@ -449,16 +449,19 @@
 ompile(%22
+%C2%A5%C2%A5s
 %22);%0A    
@@ -1427,24 +1427,229 @@
 ction) -%3E %7B%0A
+                        Matcher skipMatcher = skipPattern.matcher(next);%0A                        skipMatcher.find(0);%0A                        next = next.substring(skipMatcher.start(), skipMatcher.end());%0A
             

Handle spacing