def unique(data_dir, files):
    """
    Generate set of unique words used in both transcriptions and ground truth.

    :param data_dir: the root directory of the transcript files
    :param files: a list of file names
    :return: a set of unique words in all files.
    """
    words = []
    for file in files:
        with open(data_dir + file + '.txt', 'r') as f:
            # Get all lines. Add transcriptions words. Add ground truth words.
            lines = [line.strip().split('\t') for line in f]
            words.extend([word for line in lines for word in
                          line[4].replace(' ', '|').split('|')])
            words.extend([word for line in lines for word in line[-1].split()])
    return set(words)