@@ -0,0 +1,2152 @@
+import gzip%0Aimport json%0A%0Afrom time import time%0Afrom ..data import Store%0A%0Aimport logging%0Alog = logging.getLogger()%0A%0Aclass BuildWikidataEntityDescriptions(object):%0A    %22%22%22 Extracts an entity-%3Eshort_description store from a wikidata dump %22%22%22%0A    def __init__(self, wikidata_dump_path):%0A        self.wikidata_dump_path = wikidata_dump_path%0A%0A    def __call__(self):%0A        EST_DESC_TOTAL = 2600000 # only used to try and predict runtime%0A        desc_count = 0%0A%0A        store = Store.Get('models:descriptions')%0A%0A        start_time = time()%0A        with store.batched_inserter(250000) as s:%0A            with gzip.open(self.wikidata_dump_path, 'r') as gzf:%0A                for i, line in enumerate(gzf):%0A                    if i == 50000 or i %25 250000 == 0:%0A                        completed = desc_count / float(EST_DESC_TOTAL)%0A                        elapsed = time() - start_time%0A                        if desc_count:%0A                            eta = ((elapsed / completed)-elapsed) / 60%0A                        else:%0A                            eta = 0%0A                        log.debug('Processed %25i objects with %25i descriptions, ~%25.1f%25%25 done. Eta=%25.2fm ...', i, desc_count, completed*100, eta)%0A                    %0A                    try:%0A                        obj = json.loads(line%5B:-2%5D)%0A                        entity = obj%5B'sitelinks'%5D%5B'enwiki'%5D%5B'title'%5D%0A                        description = obj%5B'descriptions'%5D%5B'en'%5D%5B'value'%5D%0A                    except Exception:%0A                        # json.load will fail on the first and last line of the file%0A                        # some entities will be missing english descriptions or sitelinks%0A                        continue%0A                    %0A                    if description:%0A                        desc_count += 1%0A                        s.save(%7B%0A                            '_id': entity.replace(' ','_'),%0A                            'description': description%0A                        %7D)%0A%0A        log.info('Done')%0A%0A    @classmethod%0A    def add_arguments(cls, p):%0A        p.add_argument('wikidata_dump_path', metavar='WIKIDATA_DUMP_PATH')%0A        p.set_defaults(cls=cls)%0A        return p%0A

Support for generation of short entity descriptions from wikidata