#!/usr/bin/env python 
#-*- coding: utf-8 -*- 

"""
restore memory from database, if memory failed
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 23, 2013


import sys 
reload(sys) 
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config.settings import Collection, db
from config.settings import rclient
from datetime import datetime, timedelta
from scraper import memory

# CONSTANTS
from config.settings import MEMORY_RESTORATION_DAYS
from config.settings import LANGUAGES


def _get_expiration(updated):
    """
    Compute expiration time
    """
    if not updated:
        return 0

    deadline = datetime.utcfromtimestamp(updated) + timedelta(days=MEMORY_RESTORATION_DAYS)
    time_left = deadline - datetime.now()
    return time_left.days * 60 * 60 * 24 + time_left.seconds


def restore():
    """
    if memory failed, restore items from database
    """
    # check if redis is alive
    try:
        rcling.ping()
    except ConnectionError:
        print 'Redis is down! Cannot recover memory data from database'
        return None

    print '=== RESTORE MEMORY FROM DATABASE ==='
    collections = db.collection_names()
    for collection in collections:
        if collection != 'system.indexes' and collection != 'feeds':
            language = collection
            col = Collection(db, language)

            # find valid time to filter out expired items
            current_utc_time_posix = calendar.timegm(time.gmtime())
            active_datetime = datetime.utcfromtimestamp(current_utc_time_posix) - timedelta(days=MEMORY_RESTORATION_DAYS)
            active_posix = calendar.timegm(active_datetime.timetuple())

            items = col.find({'updated': {'$gte': active_posix}}).sort('updated', -1)
            if items:
                items_with_expiration = [(item, _get_expiration(float(item['updated']))) for item in items]
                memory.update_memory(items_with_expiration, language, category, feed_id)
            print language, category, feed_id, len(items_with_expiration)
        l.close()
        print 'memory to database for %s' % language
        language_in_memory_total = rclient.zcard(language)
        language_ids = rclient.zrange(language, 0, language_in_memory_total)
        if language_ids:
            for language_id in language_ids:
                if not rclient.get(language_id):
                    print language_id, 'is missing'
                    id_aloof = col.find_one({'_id': ObjectId(language_id)})
                    if id_aloof:
                        for key, value in id_aloof.iteritems():
                            id_aloof[key] = str(value)
                        rclient.set(id_aloof['_id'], id_aloof)
                        # print '--', id_aloof['_id'],
                        # datetime.utcfromtimestamp(id_aloof['updated']),
                        # get_expiration(float(id_aloof['updated']))
                        rclient.expire(
                            id_aloof['_id'], get_expiration(float(id_aloof['updated'])))
                        print 'restored', id_aloof['_id'], id_aloof['title'], datetime.utcfromtimestamp(float(id_aloof['updated']))
        print
    return 0


if __name__ == "__main__":
    restore()

@@ -929,24 +929,37 @@
 ase%0A    %22%22%22%0A
+    try:%0A    
     # check 
@@ -976,28 +976,75 @@
 s alive%0A    
-try:
+    rclient.ping()%0A        # flushall existing data
 %0A        rcl
@@ -1048,15 +1048,20 @@
 rcli
-ng.ping
+ent.flushall
 ()%0A 
@@ -1899,81 +1899,8 @@
     
- items_with_expiration = %5B(item, _get_expiration(float(item%5B'updated'%5D)))
  for
@@ -1913,17 +1913,17 @@
 in items
-%5D
+:
 %0A       
@@ -1935,939 +1935,51 @@
     
-memory.update_memory(items_with_expiration, language, category, feed_id)%0A            print language, category, feed_id, len(items_with_expiration)%0A        l.close()%0A        print 'memory to database for %25s' %25 language%0A        language_in_memory_total = rclient.zcard(language)%0A        language_ids = rclient.zrange(language, 0, language_in_memory_total)%0A        if language_ids:%0A            for language_id in language_ids:%0A                if not rclient.get(language_id):%0A                    print language_id, 'is missing'%0A                    id_aloof = col.find_one(%7B'_id': ObjectId(language_id)%7D)%0A                    if id_aloof:%0A                        for key, value in id_aloof.iteritems():%0A                            id_aloof%5Bkey%5D = str(value)%0A                        rclient.set(id_aloof%5B'_id'%5D, id_aloof)%0A                        # print '--', id_aloof%5B'_id'%5D,%0A                        # datetime.utcfromtimestamp(id_aloof
+    expiration = _get_expiration(float(item
 %5B'up
@@ -1986,17 +1986,17 @@
 dated'%5D)
-,
+)
 %0A       
@@ -2012,344 +2012,39 @@
     
-    # get_expiration(float(id_aloof%5B'updated'%5D))%0A                        rclient.expire(%0A                            id_aloof%5B'_id'%5D, get_expiration(float(id_aloof%5B'updated'%5D)))%0A                        print 'restored', id_aloof%5B'_id'%5D, id_aloof%5B'title'%5D, datetime.utcfromtimestamp(float(id_aloof%5B'updated'%5D))%0A        print%0A    return 0
+memory.update(item, expiration)
 %0A%0A%0Ai

Remove unnecessary code