import re


def _tokenizer_pattern_from(tokens):
    """ Generates regular expression tokenizer pattern from token list."""

    tokens.sort(key=len, reverse=True)
    regex_str = "(" + "|".join([re.escape(_) for _ in tokens]) + ")"
    return regex_str