@@ -0,0 +1,2061 @@
+GEVENT = False%0A%0Afrom regs_models import *%0A%0Aimport settings%0Aimport rawes%0A%0Adef run():%0A    es = rawes.Elastic(getattr(settings, %22ES_HOST%22, 'thrift://localhost:9500'), timeout=60.0)%0A    index = getattr(es, settings.ES_INDEX)%0A%0A    records = %7B%0A        'sec_cftc': %7B%7D,%0A        'regulations.gov': %7B%7D%0A    %7D%0A%0A    for doc in Doc.objects(type__in=%5B'notice', 'proposed_rule', 'rule'%5D, agency__in=%5B'SEC', 'CFTC'%5D):%0A        # first check the annotation%0A        if 'fr_data' in doc.annotations and doc.annotations%5B'fr_data'%5D:%0A            #print 'annotation', doc.source, doc.id, doc.annotations%5B'fr_data'%5D%5B'document_number'%5D%0A            records%5Bdoc.source%5D%5Bdoc.annotations%5B'fr_data'%5D%5B'document_number'%5D%5D = doc%0A        elif 'Federal_Register_Number' in doc.details:%0A            #print 'detail', doc.source, doc.id, doc.details%5B'Federal_Register_Number'%5D%0A            frn = doc.details%5B'Federal_Register_Number'%5D%0A            # trim leading zeros from the second part%0A            if %22-%22 in frn:%0A                frnp = frn.split(%22-%22)%0A                frn = %22-%22.join(frnp%5B:-1%5D + %5Bfrnp%5B-1%5D.lstrip('0')%5D)%0A            records%5Bdoc.source%5D%5Bfrn%5D = doc%0A%0A    overlap = records%5B'sec_cftc'%5D.viewkeys() & records%5B'regulations.gov'%5D.viewkeys()%0A    for frid in overlap:%0A        winner = records%5B'sec_cftc'%5D%5Bfrid%5D%0A        loser = records%5B'regulations.gov'%5D%5Bfrid%5D%0A%0A        winner_dkt = Docket.objects.get(id=winner.docket_id)%0A        loser_dkt = Docket.objects.get(id=loser.docket_id)%0A%0A        for w, l in ((winner, loser), (winner_dkt, loser_dkt)):%0A            replaces = set(w.suppression.get('replaces', %5B%5D))%0A            replaces.add(l.id)%0A            w.suppression%5B'replaces'%5D = list(replaces)%0A%0A            replaced_by = set(l.suppression.get('replaced_by', %5B%5D))%0A            replaced_by.add(w.id)%0A            l.suppression%5B'replaced_by'%5D = list(replaced_by)%0A%0A            l.save()%0A            w.save()%0A%0A        try:%0A            index.docket.delete(loser_dkt.id)%0A            index.document.delete(loser.id)%0A        except:%0A            pass%0A%0A        print '%25s suppresses %25s' %25 (winner.id, loser.id)

Add duplicate suppressor.