@@ -0,0 +1,1899 @@
+import argparse%0Aimport csv%0Aimport hashlib%0A%0Aimport elasticsearch%0Afrom elasticsearch_dsl import Search%0Aimport tqdm%0Afrom w3lib.url import canonicalize_url%0A%0A%0Adef main():%0A    parser = argparse.ArgumentParser(%0A        description='Download item hashes from ES index')%0A    arg = parser.add_argument%0A    arg('output', help='output in .csv format')%0A    arg('index', help='ES index name')%0A    arg('--domain', help='url.domain to filter')%0A    arg('--host', default='localhost', help='ES host in host%5B:port%5D format')%0A    arg('--user', help='HTTP Basic Auth user')%0A    arg('--password', help='HTTP Basic Auth password')%0A    arg('--chunk-size', type=int, default=100, help='download chunk size')%0A%0A    args = parser.parse_args()%0A    kwargs = %7B%7D%0A    if args.user or args.password:%0A        kwargs%5B'http_auth'%5D = (args.user, args.password)%0A%0A    client = elasticsearch.Elasticsearch(%0A        %5Bargs.host%5D,%0A        connection_class=elasticsearch.RequestsHttpConnection,%0A        timeout=600,%0A        **kwargs)%0A    print(client.info())%0A%0A    search = Search(using=client, index=args.index)%0A    if args.domain:%0A        search = search.filter('term', **%7B'url.domain': args.domain%7D)%0A%0A    total = 0%0A    with tqdm.tqdm(total=search.count()) as pbar:%0A        with open(args.output, 'wt') as f:%0A            writer = csv.writer(f)%0A            for x in search.params(size=args.chunk_size).scan():%0A                total += 1%0A                pbar.update(1)%0A                x = x.to_dict()%0A                writer.writerow(%5B%0A                    x%5B'timestamp_crawl'%5D,%0A                    (hashlib.sha1((x%5B'raw_content'%5D or '')%0A                     .encode('utf8')).hexdigest()),%0A                    x%5B'team'%5D,%0A                    x%5B'url'%5D,%0A                    canonicalize_url(x%5B'url'%5D, keep_fragments=True),%0A                %5D)%0A%0A    print('%7B:,%7D items downloaded to %7B%7D'.format(total, args.output))%0A%0A%0Aif __name__ == '__main__':%0A    main()%0A

Add a script that downloads content hashes