# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs

class ZhihuPipeline(object):
    def process_item(self, item, spider):
        return item



import json

class JsonWriterPipeline(object):

    def __init__(self):
        self.file = codecs.open('items.json', 'w', encoding="utf-8")

    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(line)
        return item


import pymongo

from scrapy.conf import settings

class MongoDBPipeline(object):

    collection_name = settings['MONGO_COLLECTION']

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'circle')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item))
        return item
@@ -643,16 +643,55 @@
 settings
+%0Afrom scrapy.exceptions import DropItem
 %0A%0Aclass 
@@ -1230,16 +1230,72 @@
 ongo_db%5D
+%0A        self.collection = self.db%5Bself.collection_name%5D
 %0A%0A    de
@@ -1403,24 +1403,231 @@
         
-self.db%5B
+matching_item = self.collection.find_one(%7B%0A            'title': item%5B'title'%5D%0A        %7D)%0A        if matching_item is not None:%0A            raise DropItem(%22Duplicate item found: %25s%22 %25 item)%0A        else:%0A            
 self.col
@@ -1633,22 +1633,16 @@
 llection
-_name%5D
 .insert(

drop item when duplicate