def load_data(data_dir, files):
    """
    Loads all references sentences and all hypothesis sentences for all the
    given data files in the given directory. Returns two aligned lists.

    :param data_dir: the root directory of the transcript files
    :param files: a list of file names
    :return: references_list, hypothesis_list
    """
    refs, hyps = [], []
    for file in files:
        with open(data_dir + file + '.txt', 'r', encoding = 'utf8') as f:
            lines = [line.strip().split('\t') for line in f]
            hyps.extend([line[4].split('|')[0] for line in lines])
            refs.extend([line[-1] for line in lines])
    # Remove all empty hypotheses.
    refs, hyps = [r for r, h in zip(refs, hyps) if h], [h for h in hyps if h]
    return refs, hyps