@@ -0,0 +1,3221 @@
+import os%0Aimport sys%0Aimport time%0Aimport logging.config%0Aimport json%0A%0Aclass Utils:%0A%0A    @staticmethod%0A    def insert_record(client, dataset_id, table_id, record):%0A         %0A        result = client.push_rows(dataset_id, table_id, %5Brecord%5D, None) %0A         %0A        if result.get('insertErrors', None):%0A    %0A            print %22Record: %25s%22 %25 (json.dumps(record))%0A            print %22Error result: %25s%22 %25 result%0A            %0A            return False%0A        %0A        return True%0A%0A    @staticmethod    %0A    def import_from_file(client, dataset_id, table_id, filename, single_tweet=False):%0A        %0A        if single_tweet:%0A            %0A            record = json.loads(Utils.read_file(SAMPLE_TWEET_FILE))%0A            success = Utils.insert_record(client, dataset_id, table_id, record)%0A            return success%0A%0A        row = 0%0A        with open(filename, %22r%22) as f:%0A             %0A            for tweet in f:%0A                 %0A                record = json.loads(tweet)%0A     %0A                # ignore delete records for now            %0A                if record.get(%22delete%22, None):%0A                    continue%0A                 %0A                record_scrubbed = Utils.scrub(record)%0A                success = Utils.insert_record(client, dataset_id, table_id, record_scrubbed)%0A                if not success:%0A                    print %22Failed row: %25s %25s%22 %25 (row, json.dumps(record))%0A                    return%0A                else:%0A                    print %22Processed row: %25s%22 %25 row%0A                 %0A                row = row + 1%0A%0A    @staticmethod%0A    def scrub(d):%0A    %0A        # d.iteritems isn't used as you can't del or the iterator breaks.%0A        for key, value in d.items():%0A            %0A            if value is None:%0A                del d%5Bkey%5D%0A            elif key == 'coordinates':%0A                del d%5Bkey%5D%0A            elif key == 'attributes': # in 'place' object %0A                del d%5Bkey%5D%0A            elif key == 'bounding_box': # in 'place' object%0A                del d%5Bkey%5D%0A            elif key == 'retweeted_status':%0A                del d%5Bkey%5D%0A            elif key == 'created_at':%0A                d%5Bkey%5D = Utils.convert_timestamp(value)%0A            elif isinstance(value, dict):%0A                Utils.scrub(value)%0A        return d  # For convenience%0A    %0A    @staticmethod%0A    def convert_timestamp(str):%0A        %0A        ts = time.strptime(str,'%25a %25b %25d %25H:%25M:%25S +0000 %25Y')%0A        ts = time.strftime('%25Y-%25m-%25d %25H:%25M:%25S', ts)%0A        %0A        return ts%0A    %0A    @staticmethod%0A    def read_file(fn):%0A        %0A        data = %22%22%0A        with open(fn, %22r%22) as f:%0A            for line in f:%0A                data = data + line%0A                %0A        return data    %0A    %0A    @staticmethod %0A    def generate_schema_from_tweet():%0A        %0A        record_str = Utils.read_file(SAMPLE_TWEET_FILE)%0A        record = json.loads(record_str)%0A        schema_str = schema_from_record(record)%0A        %0A        return schema_str%0A    %0A    @staticmethod%0A    def enable_logging():%0A        LOGGING_CONFIG = os.path.join(os.path.dirname(__file__),  %22logging.conf%22)%0A        print %22LOGGING_CONFIG%22 + str(LOGGING_CONFIG)%0A        logging.config.fileConfig(LOGGING_CONFIG)%0A        root = logging.getLogger(%22root%22)%0A        return root%0A

Create utils.py