import json

from bigquery import get_client
from bigquery import schema_from_record
from config import *

import logging
import sys

root = logging.getLogger()
root.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)

# record = None
# with open (RECORD_FILE, "r") as myfile:
#     data = myfile.read()
#     record = json.loads(data)
#  
# # create schema
# schema = schema_from_record(record)
# 
# print "schema: %s" % schema

# get client
client = get_client(PROJECT_ID, service_account=SERVICE_ACCOUNT, private_key=KEY, readonly=False)
client.swallow_results = False

print "client: %s" % client

# create table
created = client.create_table(DATASET_ID, TABLE_ID, SCHEMA)
 
print "created: %s" % created
 
with open(RECORDS_FILE, 'rU') as f:
    for line in f:
        record = json.loads(line)
        print "record: %s" % record
        inserted = client.push_rows(DATASET_ID, TABLE_ID, [record], None) 
        print "inserted: %s" % inserted
 
# # Submit an async query.
# job_id, _results = client.query('SELECT * FROM dataset.my_table LIMIT 1000')
# 
# # Check if the query has finished running.
# complete, row_count = client.check_job(job_id)
# 
# # Retrieve the results.
# results = client.get_query_rows(job_id)
@@ -388,48 +388,37 @@
 h)%0A%0A
-# record = None%0A# with open (RECORD_FILE
+schema = %22%22%0Awith open (SCHEMA
 , %22r
@@ -427,140 +427,73 @@
  as 
-myfile
+f
 :%0A
-#
     
- data = myfile.read()%0A#     record = json.loads(data)%0A#  %0A# # create schema%0A# schema = schema_from_record(record)%0A#
+schema = schema + f.read()%0Aschema = json.loads(schema)%0A 
  %0A
-# 
 prin
@@ -756,22 +756,22 @@
 BLE_ID, 
-SCHEMA
+schema
 )%0A %0Aprin
@@ -796,17 +796,27 @@
 created%0A
- 
+%0Atweet = %22%22
 %0Awith op
@@ -836,12 +836,11 @@
 LE, 
-'rU'
+%22r%22
 ) as
@@ -870,16 +870,38 @@
         
+tweet = tweet + line%0A%0A
 record =
@@ -916,22 +916,15 @@
 ads(
-line)%0A        
+tweet)%0A
 prin
@@ -947,24 +947,16 @@
  record%0A
-        
 inserted
@@ -1014,24 +1014,16 @@
  None) %0A
-        
 print %22i

create schema from file