@@ -0,0 +1,2318 @@
+# -*- coding: utf-8 -*-%0Aimport scrapy%0Afrom scrapy.linkextractors import LinkExtractor%0Afrom scrapy.spiders import CrawlSpider, Rule%0A%0Afrom corpus_builder.items import TextEntry%0A%0A%0Aclass BhorerkagojSpider(CrawlSpider):%0A    name = 'bhorerkagoj'%0A    allowed_domains = %5B'bhorerkagoj.net'%5D%0A    start_urls = %5B'http://www.bhorerkagoj.net/online/'%5D%0A%0A    rules = (%0A        Rule(LinkExtractor(%0A            allow='%5C/%5Cd%7B4%7D%5C/%5Cd%7B2%7D%5C/%5Cd%7B2%7D%5C/%5Cd+%5C.php$'%0A        ),%0A        callback='parse_news'),%0A    )%0A%0A    def __init__(self, start_page=None, end_page=None, category=None, *a, **kw):%0A%0A        if not (start_page or end_page):%0A            raise ValueError(%22start_page, end_page must be provided as arguments%22)%0A%0A        self.start_page = int(start_page)%0A        if end_page:%0A            self.end_page = int(end_page)%0A        else:%0A            self.end_page = self.start_page%0A%0A        self.category = None%0A        %0A        if category:%0A            self.category = category%0A        %0A        super(BhorerkagojSpider, self).__init__(*a, **kw)%0A%0A    def start_requests(self):%0A        yield scrapy.Request('http://bhorerkagoj.net/online',%0A                             callback = self.start_categorized_requests)%0A%0A    def start_categorized_requests(self, response):%0A        categories = %5B%5D%0A        if not self.category:%0A            categories = list(set(response.css('#navcatlist a::attr(%22href%22)').re('(?%3C=category/).*')))%0A        else:%0A            categories = response.css('#navcatlist a::attr(%22href%22)').re('category/%7B0%7D'.format(self.category))%0A            %0A        if not categories:%0A            raise ValueError('Invalid category')%0A        %0A        for category in categories:%0A            for page in range(self.start_page, self.end_page+1):%0A                yield scrapy.Request('http://bhorerkagoj.net/online/' + category + '/page/%7B0%7D'.format(str(page)),%0A                                     callback=self.start_news_requests)%0A%0A    def start_news_requests(self, response):%0A        news_links = list(set(response.css('.news-box h3 a::attr(%22href%22)').extract()))%0A%0A        for link in news_links:%0A            yield self.make_requests_from_url(link)%0A                    %0A    def parse_news(self, response):%0A        item = TextEntry()%0A        item%5B'body'%5D = %22%22.join(part for part in response.css('div.entry p::text').extract())%0A        return item%0A

add bhorer_kagoj