@@ -0,0 +1,2715 @@
+%22%22%22 Scrapy extension to export scraped items of different types to multiple feeds. %22%22%22%0A%0Aimport inspect%0Aimport importlib%0A%0Afrom datetime import datetime%0A%0Afrom twisted.internet import defer%0A%0Afrom scrapy import log%0Afrom scrapy.contrib.feedexport import FeedExporter, SpiderSlot%0A%0A__version__ = '0.1.0'%0A%0Aclass MultiFeedExporter(FeedExporter):%0A%0A    def __init__(self, settings):%0A        super( MultiFeedExporter, self ).__init__(settings)%0A        self.item_names = settings%5B'MULTIFEEDEXPORTER_ITEMS'%5D%0A        self.slots = %7B%7D%0A        self.item2uri = %7B%7D%0A%0A    @classmethod%0A    def get_bot_items(cls, bot_name):%0A        %22%22%22 Get item names from items module of the given bot module name%0A        %22%22%22%0A        item_module = importlib.import_module(bot_name + %22.items%22)%0A        item_names = %5Bname for name,obj in inspect.getmembers(item_module, inspect.isclass)%5D%0A        return item_names%0A%0A    def open_spider(self, spider):%0A        uri_slots = %7B%7D%0A        for item_name in self.item_names :%0A            up = self._get_uri_params(spider, item_name)%0A            uri = self.urifmt %25 up%0A            if uri not in self.slots: %0A                storage = self._get_storage(uri)%0A                file = storage.open(spider)%0A                exporter = self._get_exporter(file)%0A                exporter.start_exporting()%0A                self.slots%5Buri%5D = SpiderSlot(file, exporter, storage, uri)%0A            self.item2uri%5Bitem_name%5D = uri%0A%0A    def close_spider(self, spider):%0A        deferreds = %5B%5D%0A        for uri in self.slots:%0A            slot = self.slots%5Buri%5D%0A            if not slot.itemcount and not self.store_empty:%0A                continue%0A            slot.exporter.finish_exporting()%0A            logfmt = %22%25%25s %25s feed (%25d items) in: %25s%22 %25 (self.format, %5C%0A                slot.itemcount, slot.uri)%0A            d = defer.maybeDeferred(slot.storage.store, slot.file)%0A            d.addCallback(lambda _,logfmt=logfmt: log.msg(logfmt %25 %22Stored%22, spider=spider))%0A            d.addErrback(log.err, logfmt %25 %22Error storing%22, spider=spider)%0A            deferreds.append(d)%0A        %0A        if deferreds:%0A            return defer.DeferredList(deferreds)%0A%0A%0A    def item_scraped(self, item, spider):%0A        item_name = type(item).__name__%0A        slot = self.slots%5Bself.item2uri%5Bitem_name%5D%5D%0A        slot.exporter.export_item(item)%0A        slot.itemcount += 1%0A        return item        %0A%0A    def _get_uri_params(self, spider, item_name):%0A        params = %7B%7D%0A        for k in dir(spider):%0A            params%5Bk%5D = getattr(spider, k)%0A        ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')%0A        params%5B'time'%5D = ts%0A        params%5B'item_name'%5D = item_name%0A        self._uripar(params, spider)%0A        return params%0A

Add Python code