# -*- coding: utf-8 -*-
import scrapy
import re

from locations.items import GeojsonPointItem


class PublixSpider(scrapy.Spider):
    name = "publix"
    allowed_domains = ['publix.com']
    start_urls = (
        'http://weeklyad.publix.com/Publix/Entry/Locations/',
    )

    def parse(self, response):
        response.selector.remove_namespaces()
        city_urls = response.xpath('//a[@class="stdLink action-tracking-nav"]/@href').extract()
#        regex = re.compile(r'Publix\/BrowseByListing\/ByAllCategories\/\?StoreID=\d+')
        for path in city_urls:
#            if re.search(regex,path):
            yield scrapy.Request(
                str("http://weeklyad.publix.com") + path.strip(),
                callback=self.parse_ad,
            )


    def parse_ad(self, response):
        response.selector.remove_namespaces()
        if response.xpath('//h3[@class="weeklyAdsLoc_text"]/span[@class="mainLocName"]').extract_first():
            storeNumber = response.xpath('//h3[@class="weeklyAdsLoc_text"]/span[@class="mainLocName"]').extract_first().split('#')[1].split(')')[0]
            yield scrapy.Request(
                str("http://www.publix.com/locations/") + storeNumber.strip(),
                callback=self.parse_store,
            )


    def parse_store(self, response):

        if "CLOSED" is response.xpath('//span[@class="store-status"]/text()').extract_first():
            storeHours = 'STORE CLOSED'

        else:
            storeHoursHTML = response.xpath('//div[@class="store-info-group"]').extract()[4]
            p = re.compile(r'<.*?>')
            storeHours = p.sub('', storeHoursHTML)
            storeHours = storeHours.replace('\t', '').replace('\r', '').replace('\n', '').replace('       ', ' ')
            storeHours = "".join(storeHours.strip())


        properties = {
        'name': response.xpath('//h1[@id="content_2_TitleTag"]/text()').extract_first(),
        'ref': response.xpath('//div[@class="store-info-group"]/text()').extract_first(),
        'addr_full': response.xpath('///div[@class="store-info-group"][2]/text()').extract_first(),
        'city': response.xpath('///div[@class="store-info-group"][2]/text()').extract()[1].split(',')[0],
        'state': response.xpath('//div[@class="store-info-group"]/text()').extract()[2].split('\xa0')[0].split('\t')[-1],
        'postcode': response.xpath('///div[@class="store-info-group"][2]/text()').extract()[1].split(',')[1].split()[1],
        'phone': response.xpath('///div[@class="store-info-group"]/div[1]/text()').extract_first(),
        'website': response.request.url,
        'opening_hours': storeHours,
        'lat': response.xpath('///div[@class="store-info-group"][4]/a/@href').extract_first().split('//')[2].split(',')[0],
        'lon': response.xpath('///div[@class="store-info-group"][4]/a/@href').extract_first().split('//')[2].split(',')[1],
        }

        print(str(properties))

        yield GeojsonPointItem(**properties)
@@ -2892,39 +2892,8 @@
  %7D%0A%0A
-        print(str(properties))%0A
 %0A   

Add files via upload