#!/usr/bin/env python 
#-*- coding: utf-8 -*- 

"""
restore memory from database, if memory failed
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 23, 2013


import sys 
reload(sys) 
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config.settings import Collection, db
from config.settings import rclient
from scraper import memory

# CONSTANTS
from config.settings import MEMORY_RESTORATION_DAYS


def restore():
    '''if memory failed, restore items from database'''
    def get_expiration(updated):
        ''''''
        deadline = datetime.utcfromtimestamp(
            updated) + timedelta(days=MEMORY_RESTORATION_DAYS)
        time_left = deadline - datetime.now()
        return time_left.days * 60 * 60 * 24 + time_left.seconds

    print '----------------------restoring-------------------------'
    language_files = read_task_directory()
    for language_file in language_files:
        print 'database to memory'
        l = open(language_file, 'r')
        lines = l.readlines()
        language = None
        col = None
        for line in lines:
            language, category, feed_id, feed_link = extract_task(line)
            col = Collection(db, language)
            current_utc_time_posix = calendar.timegm(time.gmtime())
            active_datetime = datetime.utcfromtimestamp(
                current_utc_time_posix) - timedelta(days=MEMORY_RESTORATION_DAYS)
            active_posix = calendar.timegm(active_datetime.timetuple())
            items = col.find(
                {'category': category, 'feed': feed_id, 'updated': {'$gte': active_posix}}).sort('updated', -1)
            if items:
                items_with_expiration = [(item, get_expiration(float(item['updated'])))
                                         for item in items]
                # for i in items_with_expiration:
                    #ent = i[0]
                    #exp = i[1]
                    # print '++', ent['_id'],
                    # datetime.utcfromtimestamp(ent['updated']), exp
                memory.update_memory(
                    items_with_expiration, language, category, feed_id)
            print language, category, feed_id, len(items_with_expiration)
        l.close()
        print 'memory to database for %s' % language
        language_in_memory_total = rclient.zcard(language)
        language_ids = rclient.zrange(language, 0, language_in_memory_total)
        if language_ids:
            for language_id in language_ids:
                if not rclient.get(language_id):
                    print language_id, 'is missing'
                    id_aloof = col.find_one({'_id': ObjectId(language_id)})
                    if id_aloof:
                        for key, value in id_aloof.iteritems():
                            id_aloof[key] = str(value)
                        rclient.set(id_aloof['_id'], id_aloof)
                        # print '--', id_aloof['_id'],
                        # datetime.utcfromtimestamp(id_aloof['updated']),
                        # get_expiration(float(id_aloof['updated']))
                        rclient.expire(
                            id_aloof['_id'], get_expiration(float(id_aloof['updated'])))
                        print 'restored', id_aloof['_id'], id_aloof['title'], datetime.utcfromtimestamp(float(id_aloof['updated']))
        print
    return 0


if __name__ == "__main__":
    restore()

@@ -370,16 +370,57 @@
 rclient%0A
+from datetime import datetime, timedelta%0A
 from scr
@@ -507,89 +507,53 @@
 AYS%0A
-%0A%0Adef restore():%0A    '''if memory failed, restore items from database'''%0A    
+from config.settings import LANGUAGES%0A%0A%0A
 def 
+_
 get_
@@ -581,23 +581,86 @@
     
-    ''''''%0A    
+%22%22%22%0A    Compute expiration time%0A    %22%22%22%0A    if not updated:%0A        return 0%0A%0A
     
@@ -696,29 +696,16 @@
 mestamp(
-%0A            
 updated)
@@ -743,28 +743,24 @@
 ATION_DAYS)%0A
-    
     time_lef
@@ -789,20 +789,16 @@
 e.now()%0A
-    
     retu
@@ -855,447 +855,600 @@
 ds%0A%0A
-    print '----------------------restoring-------------------------'%0A    language_files = read_task_directory()%0A    for language_file in language_files:%0A        print 'database to memory'%0A        l = open(language_file, 'r')%0A        lines = l.readlines()%0A        language = None%0A        col = None%0A        for line in lines:%0A            language, category, feed_id, feed_link = extract_task(line)%0A            col = Collection(db, language)
+%0Adef restore():%0A    %22%22%22%0A    if memory failed, restore items from database%0A    %22%22%22%0A    # check if redis is alive%0A    try:%0A        rcling.ping()%0A    except ConnectionError:%0A        print 'Redis is down! Cannot recover memory data from database'%0A        return None%0A%0A    print '=== RESTORE MEMORY FROM DATABASE ==='%0A    collections = db.collection_names()%0A    for collection in collections:%0A        if collection != 'system.indexes' and collection != 'feeds':%0A            language = collection%0A            col = Collection(db, language)%0A%0A            # find valid time to filter out expired items
 %0A   
@@ -1568,33 +1568,16 @@
 mestamp(
-%0A                
 current_
@@ -1702,24 +1702,25 @@
 imetuple())%0A
+%0A
             
@@ -1740,65 +1740,9 @@
 ind(
-%0A                %7B'category': category, 'feed': feed_id, 
+%7B
 'upd
@@ -1863,16 +1863,17 @@
 %5B(item, 
+_
 get_expi
@@ -1907,356 +1907,65 @@
 %5D)))
-%0A                                         for item in items%5D%0A                # for i in items_with_expiration:%0A                    #ent = i%5B0%5D%0A                    #exp = i%5B1%5D%0A                    # print '++', ent%5B'_id'%5D,%0A                    # datetime.utcfromtimestamp(ent%5B'updated'%5D), exp%0A                memory.update_memory(%0A                    
+ for item in items%5D%0A                memory.update_memory(
 item

Update restore.....py