def tokenize_lines_to_ids(lines, tokenizer):
    """
    Worker function to tokenize lines based on the tokenizer, and perform vocabulary lookup.

    Parameters
    ----------
    lines
        Lines to be tokenized of the whole file
    tokenizer
        The trained tokenizer

    Returns
    -------
    results
        A list storing the valid tokenized lines
    """
    results = []
    # tag line delimiters or doc delimiters
    for line in lines:
        if not line:
            break
        line = line.strip()
        # Single empty lines are used as line delimiters
        # Double empty lines are used as document delimiters
        if not line:
            results.append([])
        else:
            token_ids = tokenizer.encode(line, int)
            if token_ids:
                results.append(token_ids)
    return results