#!/usr/bin/env python

"""
No description.
"""

import sys
from scrapy import cmdline, Request
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

TARGET_DOMAIN = 'hnalady.com'

class HNaLadyCrawler(CrawlSpider):
    """No descrition"""

    name = TARGET_DOMAIN
    allowed_domains = [TARGET_DOMAIN]
    custom_settings = {
        'DOWNLOAD_DELAY': 1,
    }

    rules = (
        Rule(LinkExtractor(restrict_xpaths='(//div[@id="main-right"]/div[@class="page_navi"]/p[@class="page_next"]/a)[1]')),
        Rule(LinkExtractor(restrict_xpaths='//h3/a[contains(@title, "このエントリーの固定リンク")]'), callback='parse_entry'),
    )

    def start_requests(self):
        """No descrition"""

        url = self.tag
        yield Request(url, dont_filter=True)

    def parse_entry(self, response):
        """No descrition"""
        images = response.xpath('//div[@id="more"]/a/img/@src')

        # Recommend -O images.csv
        for image in images:
            yield {'image': image.get() for image in images}


if __name__ == '__main__':
    #cmdline.execute(f"scrapy runspider {sys.argv[0]} -a tag={sys.argv[1]} -O images.csv".split())
    command_line = ["scrapy", "runspider"]
    command_line.extend(sys.argv)
    cmdline.execute(command_line)

@@ -849,24 +849,25 @@
 tion%22%22%22%0A
+%0A
         
 images =
@@ -862,15 +862,19 @@
     
+if 
 images 
+:
 = re
@@ -921,134 +921,175 @@
 rc')
-%0A%0A        # Recommend -O images.csv%0A        for image in images:%0A            yield %7B'image': image.get() for 
+.getall():%0A            yield %7B%0A                'title': response.xpath('//title/text()').get(),%0A                'url': response.url,%0A                '
 image
- in
+s':
  images
+ 
 %7D
-%0A
 %0A%0Aif

fix emerging items