#!/usr/bin/python
import time
from elasticsearch import Elasticsearch
from datetime import datetime
from pymongo import MongoClient


es = Elasticsearch()
client = MongoClient('mongodb://localhost:27017/')


def index(index_name):
    db = client['v2ex']['topic'] 

    for item in db.find():
        item['created'] = item['created'] * 1000
        item['last_modified'] = item['last_modified'] * 1000
        item['last_touched'] = item['last_touched'] * 1000
        item['rcontent'] = get_replies(item['_id'])
        try:
            es.index(index=index_name, doc_type="topic", id=item['_id'], body=item)
        except Exception, e:
            print(e)
            time.sleep(5)
       
        info = "indexed topic %d"%(item['_id'])
        print(info)


def get_replies(topic_id):

    """
        return replies like:
            'username created content  [username created content ...]'
    """

    rcontent = u""
    db = client['v2ex']['reply']
    for r in db.find({"topic_id":topic_id}):
        rcontent += r['member']['username']     
        rcontent += " " + datetime.fromtimestamp(r['created']).strftime('%Y-%m-%d')
        rcontent += " " + r['content'] + "    "

    return rcontent


if __name__ == '__main__':
    index('v2_1')


@@ -126,16 +126,25 @@
 Client%0A%0A
+START = 1
 %0Aes = El
@@ -271,34 +271,86 @@
 '%5D %0A
-%0A
     
-for item in db.find()
+cursor = db.find(%7B%22_id%22:%7B%22$gt%22:START%7D%7D).sort('_id')%0A    for item in cursor
 :%0A  
@@ -741,16 +741,37 @@
 leep(5)%0A
+            continue%0A
        %0A

make index stronger