# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import redis


from scrapy import signals


import json
import codecs
from collections import OrderedDict



class TXTWithEncodingPipeline(object):

    def __init__(self):
        self.file = codecs.open('data_utf8.txt', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        # line = '\t'.join([v for k, v in OrderedDict(item).items()]) + "\n"
        line = ''
        for k, v in OrderedDict(item).items():
            for d in v:
                # print type(d), d
                li = ['|'.join(v1) for k1, v1 in OrderedDict(d).items()]
                line = '\t'.join(li[-1:]) + '\n'
                self.file.write(line)
        return item

    def close_spider(self, spider):
        self.file.close()


class JsonWithEncodingPipeline(object):

    def __init__(self):
        self.file = codecs.open('data_utf8.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        line = json.dumps(OrderedDict(item), ensure_ascii=False, sort_keys=False) + "\n"
        self.file.write(line)
        return item

    def close_spider(self, spider):
        self.file.close()


class RedisPipeline(object):

    def __init__(self):
        self.r = redis.StrictRedis(host='localhost', port=6379)

    def process_item(self, item, spider):
        if not item['id']:
            print 'no id item!!'

        str_recorded_item = self.r.get(item['id'])
        final_item = None
        if str_recorded_item is None:
            final_item = item
        else:
            ritem = eval(self.r.get(item['id']))
            final_item = dict(item.items() + ritem.items())
        self.r.set(item['id'], final_item)

    def close_spider(self, spider):
        return

@@ -1865,8 +1865,919 @@
  return%0A
+%0Aimport sys%0Aimport MySQLdb%0Aimport hashlib%0Afrom scrapy.exceptions import DropItem%0Afrom scrapy.http import Request%0A%0Aclass MySQLStorePipeline(object):%0A  def __init__(self):%0A    # self.conn = MySQLdb.connect(user='user', 'passwd', 'dbname', 'host', charset=%22utf8%22, use_unicode=True)%0A    self.conn = MySQLdb.connect(user='dbuser', passwd='dbpasswd', db='dbname', host='dbhost', charset=%22utf8%22, use_unicode=True)%0A    self.cursor = self.conn.cursor()%0A%0A    def process_item(self, item, spider):    %0A        try:%0A            self.cursor.execute(%22%22%22INSERT INTO example_book_store (book_name, price)  %0A                            VALUES (%25s, %25s)%22%22%22, %0A                           (item%5B'book_name'%5D.encode('utf-8'), %0A                            item%5B'price'%5D.encode('utf-8')))%0A%0A            self.conn.commit()%0A%0A%0A        except MySQLdb.Error, e:%0A            print %22Error %25d: %25s%22 %25 (e.args%5B0%5D, e.args%5B1%5D)%0A%0A%0A        return item%0A

add mysql pipeline