'''
This file get eid from db,
 and add 'eid' field to each json object of each comment

Usage: python geteid.py database user password source
       source could be: imdb, MF

Note: Make sure you have already get sentiment score before running this script

Author: Wenjun Wang
Date: May 29, 2015
'''
import MySQLdb as mdb
import sys
import glob
import json
import codecs

#cur = None
def walk_through(fpath,cur):
	files = glob.glob(fpath)
	for name in files:
		print name
		try:
			json_file = codecs.open(name,encoding='utf-8')
			comments = json.load(json_file)
			to_write = codecs.open(name,'w',encoding='utf-8')
			for i in xrange(0,len(comments)):
				entity = comments[i]
				#print str(len(entity))
				for j in xrange(0,len(entity)):
					comment = entity[j]
					#print str(len(comment))
					source = comment['source']
					#print source
					identity = comment['id']
					query = 'SELECT eid FROM entity WHERE \
					source = "' + source + '" AND id = "' + identity + '";'
					#print query
					cur.execute(query)
					row = cur.fetchall()
					comment['eid'] = row[0][0]
			json.dump(comments,to_write)	
		except Exception as exc:
			print "exception in file: %s" % name
			raise

def main():
	if len(sys.argv) < 4:
		print "Usage: python geteid.py db usr pwd source"
		print "source may be: imdb, MF"
		sys.exit(1)

	database = sys.argv[1]
	usr = sys.argv[2]
	pwd = sys.argv[3]
	source = sys.argv[4]
	directory = "/home/mingf/comment_full/" + source + "_comments_*.json"
	print directory

	db = mdb.connect(host="localhost",user=usr,passwd=pwd,db=database)
	cur = db.cursor()
	
	walk_through(directory,cur)

	db.close()

if __name__ == '__main__':
	main()

@@ -135,16 +135,34 @@
 d source
+ startdate enddate
 %0A       
@@ -186,16 +186,61 @@
 imdb, MF
+%0A       date in the format like: '2015-01-01'
 %0A%0ANote: 
@@ -394,20 +394,8 @@
 sys%0A
-import glob%0A
 impo
@@ -420,95 +420,265 @@
 ecs%0A
-%0A#cur = None%0Adef walk_through(fpath,cur):%0A%09files = glob.glob(fpath)%0A%09for name in files:
+from dateutil.rrule import rrule,DAILY%0Afrom datetime import datetime%0A%0Adef walk_through(directory,cur,start,end):%0A%09for dt in rrule(DAILY,dtstart = start, until = end):%0A%09%09thisdate = dt.strftime('%25Y-%25m-%25d')%0A%09%09name = '%25s_comments_%25s.json' %25 (directory,thisdate)
 %0A%09%09p
@@ -1495,16 +1495,34 @@
 d source
+ startdate enddate
 %22%0A%09%09prin
@@ -1549,16 +1549,62 @@
 db, MF%22%0A
+%09%09print %22Date in the format like: 2015-01-01%22%0A
 %09%09sys.ex
@@ -1747,31 +1747,125 @@
 urce
- + %22_comments_*.json%22
+%0A%09start = datetime.strptime(sys.argv%5B5%5D,'%25Y-%25m-%25d').date()%0A%09end = datetime.strptime(sys.argv%5B6%5D,'%25Y-%25m-%25d').date()
 %0A%09
+#
 prin
@@ -1993,18 +1993,44 @@
 tory,cur
-)%0A
+,start,end)%0A%0A%09cur.close()%09%09%09
 %0A%09db.clo

Allow daily update