import boto
import datetime
import uuid
import json

import os
import errno
import sys
import pandas as pd
import time

from sqlalchemy import create_engine

class PostgresDataIngestSink:
  """Output data to Postgres"""

  def __init__(self, config):
    self.config = config

    self.connection_string = self.config['connection_string']
    self.table_name = self.config['table_name']
    self.engine = create_engine(self.connection_string)

    print(
      '[ProgresSink] Writing to connection string ' + 
      self.connection_string + ' table: ' + self.table_name
    )

    self.records_written = 0
    self.batch_size = 50


  def write(self, source):
    for item in source:
      try:
        tweet = [ item ]

        #print 'Tweet: ' + str(tweet)

        createds = [time.strftime('%Y-%m-%d %H:%M:%S',
          time.strptime(str(tweet[i]['tweet']['created_at']),
              '%a %b %d %H:%M:%S +0000 %Y'))                  for i in range(len(tweet))]
        userid =   [tweet[i]['tweet']['id']                         for i in range(len(tweet))]
        texts =    [tweet[i]['tweet']['text']                       for i in range(len(tweet))]
        retweets = [tweet[i]['tweet']['retweet_count']              for i in range(len(tweet))]
        follows =  [tweet[i]['tweet']['user']['followers_count']    for i in range(len(tweet))]
        friends_count = [tweet[i]['tweet']['user']['friends_count'] for i in range(len(tweet))]
        urls =     [tweet[i]['tweet']['user']['url'] for i in range(len(tweet))]

        df = pd.DataFrame(
          {'created_at':createds, 'userid':userid, 'retweets':retweets,
            'text':texts, 'friendcount':friends_count,
                     'followers': follows, 'urls': urls},
          columns=['created_at','userid','retweets', 'text','friendcount', 'followers', 'urls']
        )

        #print 'Read: ' + str(df)

        df.to_sql(self.table_name, self.engine, if_exists = 'append')


        sys.stdout.write('.') # write a record indicator to stdout
      except:
        sys.stdout.write('*')

      sys.stdout.flush()
      self.records_written = self.records_written + 1

      if self.records_written % self.batch_size == 0:
        print('|')

      #raise ValueError('time to stop')


    

    

@@ -688,16 +688,46 @@
    try:%0A
+        if 'tweet' in item:%0A  
         
@@ -744,16 +744,18 @@
 item %5D%0A%0A
+  
         
@@ -793,16 +793,18 @@
         
+  
 createds
@@ -852,16 +852,18 @@
         
+ 
+ 
 time.str
@@ -906,16 +906,18 @@
 _at'%5D),%0A
+  
         
@@ -1006,16 +1006,18 @@
         
+ 
+ 
 userid =
@@ -1088,32 +1088,34 @@
 ge(len(tweet))%5D%0A
+  
         texts = 
@@ -1202,16 +1202,18 @@
         
+  
 retweets
@@ -1284,32 +1284,34 @@
 ge(len(tweet))%5D%0A
+  
         follows 
@@ -1398,16 +1398,18 @@
         
+  
 friends_
@@ -1496,16 +1496,18 @@
         
+ 
+ 
 urls =  
@@ -1572,16 +1572,18 @@
 eet))%5D%0A%0A
+  
         
@@ -1611,16 +1611,18 @@
         
+  
 %7B'create
@@ -1687,16 +1687,18 @@
         
+ 
+ 
 'text':t
@@ -1753,16 +1753,18 @@
         
+  
 'followe
@@ -1802,16 +1802,18 @@
         
+ 
+ 
 columns=
@@ -1890,16 +1890,18 @@
 'urls'%5D%0A
+  
         
@@ -1903,32 +1903,34 @@
      )%0A%0A        
+  
 #print 'Read: ' 
@@ -1940,16 +1940,18 @@
 tr(df)%0A%0A
+  
         
@@ -2012,16 +2012,59 @@
 ppend')%0A
+        elif 'post' in item:%0A          pass
 %0A%0A      

Handle tweets and posts in postgres sink