@@ -0,0 +1,1211 @@
+# -*- coding: utf-8 -*-%0A%22%22%22Script to measure title length statisics%22%22%22%0Afrom itertools import cycle%0Aimport numpy as np%0Afrom iscc_bench.readers import ALL_READERS%0A%0A%0Adef iter_titles():%0A    %22%22%22Iterate over titles%22%22%22%0A    readers = %5Br() for r in ALL_READERS%5D%0A    for reader in cycle(readers):%0A        meta = next(reader)%0A        yield meta.title%0A%0A%0Adef reject_outliers(data, m=2.):%0A    %22%22%22Remove outliers from data.%22%22%22%0A    d = np.abs(data - np.median(data))%0A    mdev = np.median(d)%0A    s = d/mdev if mdev else 0.%0A    return data%5Bs %3C m%5D%0A%0A%0Aif __name__ == '__main__':%0A%0A    SAMPLE_SIZE = 1000000%0A%0A    title_sizes = %5B%5D%0A    for n, title in enumerate(iter_titles()):%0A        title_sizes.append(len(title))%0A        if n %3E SAMPLE_SIZE:%0A            break%0A%0A    data = np.array(title_sizes, dtype=np.uint16)%0A    abs_max = max(data)%0A    print('Longest title in %7B%7D samples had %7B%7D chars.'.format(SAMPLE_SIZE, abs_max))%0A    print('The mean title length of all titles is %7B%7D chars '.format(data.mean()))%0A%0A    cleaned = reject_outliers(data)%0A    max_real = max(cleaned)%0A    print('The longest title without outliers is %7B%7D chars.'.format(max_real))%0A    print('The mean title length without outliers is %7B%7D chars.'.format(cleaned.mean()))%0A%0A

Add script to measure title length statistics