import re
import json

from scripts.migrate_piwik import utils
from scripts.migrate_piwik import settings


def main():

    input_filename = '/'.join([utils.get_dir_for('transform01'), settings.TRANSFORM01_FILE,])
    input_file = open(input_filename, 'r')

    run_id = utils.get_history_run_id_for('transform01')
    complaints_file = utils.get_complaints_for('transform01', 'w')
    complaints_file.write('Run ID: {}\n'.format(run_id))

    linenum = 0
    complaints = 0
    for pageview_json in input_file.readlines():
        linenum += 1
        if not linenum % 100:
            print('Validating line {}'.format(linenum))

        pageview = json.loads(pageview_json)

        if pageview['page']['url'] is None:
            complaints += 1
            complaints_file.write('Line {}: empty url!\n'.format(linenum))

        # if pageview['page']['title'] is None:
        #     complaints += 1
        #     complaints_file.write('Line {}: empty page title!\n'.format(linenum))

        if pageview['time']['utc'] is None:
            complaints += 1
            complaints_file.write('Line {}: missing timestamp!\n'.format(linenum))

        if pageview['tech']['ip'] is not None:
            if pageview['anon']['continent'] is None or pageview['anon']['country'] is None:
                complaints += 1
                complaints_file.write(
                    'Line {}: Have IP addr ({}), but missing continent and/or country: ({} / {})\n'.format(
                        linenum, pageview['tech']['ip'], pageview['anon']['continent'] or 'None',
                        pageview['anon']['country'] or 'None'
                    )
                )

    if complaints > 0:
        print("I got {} reasons to be mad at you.  ".format(complaints))
    else:
        print("You've done your homework, have a cookie!");

if __name__ == "__main__":
    main()

@@ -118,146 +118,8 @@
 ):%0A%0A
-    input_filename = '/'.join(%5Butils.get_dir_for('transform01'), settings.TRANSFORM01_FILE,%5D)%0A    input_file = open(input_filename, 'r')%0A%0A
     
@@ -339,53 +339,165 @@
     
-for pageview_json in input_file.readlines():%0A
+with open(utils.get_dir_for('transform01') + '/' + settings.TRANSFORM01_FILE, 'r') as input_file:%0A        for i, pageview_json in enumerate(input_file):%0A    
     
@@ -508,18 +508,21 @@
 linenum 
-+
 =
+ i +
  1%0A     
@@ -524,16 +524,20 @@
         
+    
 if not l
@@ -550,16 +550,20 @@
  %25 100:%0A
+    
         
@@ -611,32 +611,40 @@
 enum))%0A%0A        
+ 
+       
 pageview = json.
@@ -665,32 +665,40 @@
 _json)%0A%0A        
+  
+      
 if pageview%5B'pag
@@ -733,187 +733,47 @@
     
-complaints += 1%0A            complaints_file.write('Line %7B%7D: empty url!%5Cn'.format(linenum))%0A%0A        # if pageview%5B'page'%5D%5B'title'%5D is None:%0A        #     complaints += 1%0A        #
+        complaints += 1%0A               
     
@@ -815,18 +815,11 @@
 pty 
-page title
+url
 !%5Cn'
@@ -829,32 +829,40 @@
 rmat(linenum))%0A%0A
+        
         if pagev
@@ -893,32 +893,40 @@
 ne:%0A            
+  
+      
 complaints += 1%0A
@@ -917,32 +917,40 @@
 complaints += 1%0A
+        
             comp
@@ -1017,32 +1017,40 @@
 enum))%0A%0A        
+   
+     
 if pageview%5B'tec
@@ -1068,24 +1068,32 @@
 s not None:%0A
+        
             
@@ -1181,32 +1181,40 @@
                 
+     
+   
 complaints += 1%0A
@@ -1205,32 +1205,40 @@
 complaints += 1%0A
+        
                 
@@ -1280,16 +1280,24 @@
         
+      
+  
 'Line %7B%7D
@@ -1396,24 +1396,32 @@
             
+        
 linenum, pag
@@ -1498,32 +1498,40 @@
                 
+       
+ 
 pageview%5B'anon'%5D
@@ -1572,18 +1572,34 @@
         
-)%0A
+        )%0A        
         

update validate_after_transform01 to stream