import json
import os
import sys

#import dateparser
from datetime import datetime

__DATA_DIR = '../data/'

def harmonize_data( data ):
    ## make dates as date objects
    data2 = []
    for d in data:
        if 'created_time' in d:
           #d['date'] = dateparser.parse( d['created_time'] ) ## should take care of the various formats

           d['date'] = datetime.strptime( d['created_time'].replace( 'T', ' ' ).replace( '+0000', '' ), '%Y-%m-%d %H:%M:%S' )
           d['creator'] = d['from']['name']
           data2.append( d )

    return data2

def load_facebook( terms = ['data_'], data_folder = 'facebook/' ): ## todo: better filtering

    data = []

    path = __DATA_DIR + data_folder

    for f in os.listdir( path ):

        if any( term in f for term in terms ):

            #print json.load( open( __DATA_DIR + f ) ).keys()

            data += json.load( open( path + f ) )['feed']

    return harmonize_data( data )

@@ -938,8 +938,582 @@
  data )%0A
+%0Adef load_media( terms = %5B'.json'%5D, data_folder = 'media/' ):%0A%0A    import pickle ## for now, using picke - JSON later on%0A%0A    data = %5B%5D%0A%0A    path = __DATA_DIR + data_folder%0A%0A    for f in os.listdir( path ):%0A%0A        if any( term in f for term in terms ):%0A%0A            d = pickle.load( open( path + f ) )%0A%0A            ## TBA: change to standart format correctly%0A            d%5B'date'%5D = min( d%5B'datetime_list'%5D )%0A            d%5B'creator'%5D = d%5B'author'%5D%0A            d%5B'message'%5D = d%5B'title'%5D + ' ' + d%5B'ingress'%5D + ' ' + d%5B'text'%5D%0A%0A            data.append( d )%0A%0A    return data%0A

Add media data loading