#!/usr/bin/env python3

import hashlib
import json
import os
import sys
print(sys.path)
from gigacluster import Doc, dr, Tokenizer

tok = Tokenizer()
for root, dirs, files in os.walk(sys.argv[1]):
    dirs.sort()
    files = set(files)
    cluster_dr = os.path.join(root, 'cluster.dr')
    with open(cluster_dr, 'wb') as f_dr:
        writer = dr.Writer(f_dr, Doc)
        for filename in sorted(f for f in files if f.endswith('.readability')):
            dr_filename = filename.replace('.readability', '.dr')
            if not dr_filename in files:
                path = os.path.join(root, filename)
                with open(path) as f:
                    try:
                        data = json.load(f)
                    except Exception:
                        pass
                    else:
                        if not ('url' in data and 'title' in data and 'content' in data):
                            continue
                        id = hashlib.md5(data['url'].encode('utf8')).hexdigest()
                        title = data.get('title', '')
                        text = data.get('content', '')
                        doc = Doc(id=id)
                        title = title.encode('utf8')
                        tok.tokenize(title, doc, 0, is_headline=True)
                        tok.tokenize(text.encode('utf8'), doc, len(title) + 1, is_headline=False)
                        print(doc)
                        writer.write(doc)

@@ -16,17 +16,52 @@
 python3%0A
+from collections import defaultdict
 %0A
-
 import h
@@ -104,24 +104,8 @@
 sys%0A
-print(sys.path)%0A
 from
@@ -162,16 +162,41 @@
 nizer()%0A
+stats = defaultdict(int)%0A
 for root
@@ -398,24 +398,44 @@
 (f_dr, Doc)%0A
+        written = 0%0A
         for 
@@ -506,119 +506,8 @@
 )):%0A
-            dr_filename = filename.replace('.readability', '.dr')%0A            if not dr_filename in files:%0A    
     
@@ -562,20 +562,16 @@
         
-  
-  
 with ope
@@ -600,20 +600,16 @@
         
-    
 try:%0A   
@@ -625,20 +625,16 @@
         
- 
-   
 data = j
@@ -645,20 +645,16 @@
 load(f)%0A
-    
         
@@ -679,24 +679,63 @@
 eption:%0A    
+                stats%5B'bad json'%5D += 1%0A
             
@@ -767,22 +767,14 @@
     
-   
- 
 else:%0A
-    
     
@@ -855,32 +855,83 @@
  data):%0A        
+                stats%5B'no url/title/content'%5D += 1%0A
                 
@@ -934,16 +934,20 @@
         
+    
 continue
@@ -947,20 +947,16 @@
 ontinue%0A
-    
         
@@ -1036,36 +1036,32 @@
                 
-    
 title = data.get
@@ -1094,20 +1094,16 @@
         
-   
- 
 text = d
@@ -1145,20 +1145,16 @@
         
-    
 doc = Do
@@ -1174,36 +1174,32 @@
                 
-  
-  
 title = title.en
@@ -1211,20 +1211,16 @@
 'utf8')%0A
-    
         
@@ -1289,36 +1289,32 @@
                 
- 
-   
 tok.tokenize(tex
@@ -1395,17 +1395,20 @@
     
-    print
+writer.write
 (doc
@@ -1429,30 +1429,206 @@
         
-    writer.write(doc
+stats%5B'ok'%5D += 1%0A                    written += 1%0A    if not written:%0A        print('No docs written for %7B%7D'.format(cluster_dr))%0A        os.remove(cluster_dr)%0Aprint('Stats%5Ct%7B%7D'.format(dict(stats))
 )%0A

Check for empty cluster dr files.