import hashlib


def convert_data_set(data_set, data_set_writer, transformation=None,
                     count_dist=False, prime_wise=False, remove_duplicities=False):
    """
    Convert data set to a different format.

    If transformation is provided, compute features and add them to the key
    key that gets saved using a data_set_writer.

    If count_dist is True, also compute and return the distributions.

    :param data_set: source dataset.DataSet
    :param data_set_writer: target dataset.Writer
    :param transformation: optional features.Transformation
    :param count_dist: if True, compute the distributions
    :param prime_wise: iterate over primes rather than keys
    :param remove_duplicities: remove identical keys before processing
    :return: distribution for the top level transformation
             if count_dist is True, else None
    """
    feats = []
    seen_keys = set()
    for key in data_set.iterator(prime_wise=prime_wise):
        if remove_duplicities:
            digest = hashlib.sha256()
            digest.update(str(key).encode('utf-8'))
            d = digest.hexdigest()
            if d in seen_keys:
                continue
            seen_keys.add(d)
        if transformation:
            f = transformation.apply(key)
            if count_dist:
                feats.append(f)
        data_set_writer.write(key)
    if count_dist and transformation:
        return transformation.tally(feats)
    return None