@@ -0,0 +1,2684 @@
+#!/usr/bin/python3%0A%0Aimport datetime%0A%0Afrom scrapy.spiders import Spider%0Aimport pytz%0Aimport scrapy%0A%0Afrom feeds.loaders import FeedEntryItemLoader%0Afrom feeds.loaders import FeedItemLoader%0A%0A%0Aclass ProfilAtSpider(Spider):%0A    name = 'profil.at'%0A    allowed_domains = %5B'profil.at'%5D%0A%0A    _emitted_feed_item = False%0A    _datetime_format = '%25Y-%25m-%25dT%25H:%25M:%25S'%0A    _timezone = pytz.timezone('Europe/Vienna')%0A    _max_days = 3%0A%0A    def start_requests(self):%0A        now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC).astimezone(%0A            self._timezone)%0A        for day in range(self._max_days):%0A            yield scrapy.Request(%0A                'http://%7B%7D/archiv/%7B%7D'.format(%0A                    self.name, (now - datetime.timedelta(days=day)).strftime(%0A                        '%25Y/%25m/%25d')), self.parse_archive_page)%0A%0A    def parse_archive_page(self, response):%0A        if not self._emitted_feed_item:%0A            self._emitted_feed_item = True%0A            il = FeedItemLoader()%0A            il.add_value('title', 'PROFIL')%0A            il.add_value('subtitle',%0A                         '%C3%96sterreichs unabh%C3%A4ngiges Nachrichtenmagazin')%0A            il.add_value('link', 'http://www.profil.at')%0A            il.add_value('author_name', self.name)%0A            yield il.load_item()%0A%0A        for link in response.xpath('//h1/a/@href').extract():%0A            yield scrapy.Request(response.urljoin(link), self.parse_item)%0A%0A    def parse_item(self, response):%0A        remove_elems = %5B%0A            'aside', 'script', 'h1', '.breadcrumbs', '.author-date',%0A            '.artikel-social-kommentar', '.bild-copyright',%0A            '.ressortTitleMobile', '.article-number', '.artikel-kommentarlink'%0A        %5D%0A        il = FeedEntryItemLoader(response=response,%0A                                 datetime_format=self._datetime_format,%0A                                 timezone=self._timezone,%0A                                 base_url='http://%7B%7D'.format(self.name),%0A                                 remove_elems=remove_elems)%0A        il.add_value('link', response.url)%0A        il.add_xpath('author_name', '//a%5B@rel=%22author%22%5D/text()')%0A        il.add_xpath(%0A            'author_name', 'substring-before(substring-after('%0A            '(//section%5B@class=%22author-date%22%5D/text())%5B1%5D'%0A            ', %22Von%22), %22(%22)')%0A        il.add_value('author_name', 'Red.')%0A        il.add_xpath('title', '//h1/text()')%0A        il.add_xpath(%0A            'updated', 'substring-before('%0A            '//meta%5B@property=%22article:published_time%22%5D/@content'%0A            ', %22+%22)')%0A        il.add_xpath('content_html', '//article')%0A        yield il.load_item()%0A%0A# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent autoindent%0A

Add spider.