# -*- coding: utf-8 -*-
import scrapy
import re

from locations.items import GeojsonPointItem


class RubiosSpider(scrapy.Spider):
    name = "rubios"
    allowed_domains = ['rubios.com']
    start_urls = (
        'https://www.rubios.com/sitemap.xml',
    )

    def parse(self, response):
        response.selector.remove_namespaces()
        city_urls = response.xpath('//url/loc/text()').extract()
        regex = re.compile(r'http\S+rubios.com/store-locations/\S+/\S+/\S+')
        for path in city_urls:
            if re.search(regex,path):
                yield scrapy.Request(
                    path.strip(),
                    callback=self.parse_store,
                )
            else:
                pass

    def parse_store(self, response):

        properties = {
        'name': response.xpath('//span[@itemprop="name"]/text()').extract_first(),
        'ref': response.xpath('//span[@itemprop="name"]/text()').extract_first(),
        'addr_full': response.xpath('//span[@itemprop="streetAddress"]/text()').extract_first(),
        'city': response.xpath('//span[@itemprop="addressLocality"]/text()').extract_first(),
        'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
        'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
        'phone': response.xpath('//span[@itemprop="telephone"]/a/text()').extract_first(),
        'website': response.request.url,
        'opening_hours': "".join(response.xpath('//div/div/div/span/span/span/text()').extract()).strip(),
        'lon': float(response.xpath('//head/script[9]').extract_first().split('"lon":')[1].split('}')[0]),
        'lat': float(response.xpath('//head/script[9]').extract_first().split('"lat":')[1].split(',')[0]),
        }



        yield GeojsonPointItem(**properties)
@@ -1644,13 +1644,22 @@
 t('%22
-lon
+coordinates
 %22:
+%5B
 ')%5B1
@@ -1667,17 +1667,31 @@
 .split('
-%7D
+%5D')%5B0%5D.split(',
 ')%5B0%5D),%0A
@@ -1774,13 +1774,22 @@
 t('%22
-lat
+coordinates
 %22:
+%5B
 ')%5B1
@@ -1789,37 +1789,51 @@
 %22:%5B')%5B1%5D.split('
+%5D')%5B0%5D.split('
 ,')%5B
-0
+1
 %5D),%0A        %7D%0A%0A%0A

Add files via upload