from typing import List
import random


def load_paragraphs_from_files(file_name_list: List[str], shuffled: bool=True):
    """
    This function loads paragraphs from multiple files.
    A paragraph is just a string.
    It can, but does not need to contain multiple sentences.
    The function takes a list of file names for files.
    Each of these files is intended to contain multiple paragraphs
    (e.g. a list of science textbooks) with one paragraph per line.
    The function returns a list with these paragraphs, with an option to have it
    shuffled, otherwise it's sorted.
    Duplicates are removed.
    Note: Some textbook files come in the format
            chapter index  \t   paragraph.
    In this case the chapter index is removed.
    """

    # use set to avoid duplicate paragraphs
    paragraphs_collection = set()

    for filename in file_name_list:
        if '.DS_Store' in filename:
            continue
        print("reading textbook paragraphs from file ", filename, '...')
        with open(filename, 'r') as openfile:
            # expecting one paragraph per line
            paragraphs = [x.strip() for x in openfile.readlines() if x]

            # in case there's a tab-separated chapter index at the paragraph start
            if any(['\t' in p for p in paragraphs[0]]):
                # remove the chapter index
                paragraphs = [p.split('\t')[1] for p in paragraphs]

            paragraphs_collection.update(paragraphs)

    if shuffled:
        paragraphs_collection = sorted(paragraphs_collection, key=lambda x: x)
        random.shuffle(paragraphs_collection)
    return paragraphs_collection