import torch


def process_data(file_name, tokenizer, max_size=None):
    """Reads the line of a file and segments the lines using the tokenizer

    Args:
        file_name (str): Full file path
        tokenizer (:obj:`WMT16Tokenizer`): The tokenizer to use
        max_size (Optional[int]): The maximum size of result

    Returns:
        (list[int]): Tokenized data
    """
    data = []
    with open(file_name, mode="r", encoding="utf-8") as f:
        for idx, line in enumerate(f):
            if max_size and idx == max_size:
                break
            entry = tokenizer.segment(line)
            entry = torch.tensor(entry)
            data.append(entry)
    return data