# -*- coding: utf-8 -*-
import scrapy
import json
import re

from locations.items import GeojsonPointItem

class VerizonSpider(scrapy.Spider):

    name = "verizon"
    allowed_domains = ["www.verizonwireless.com"]
    start_urls = (
        'https://www.verizonwireless.com/stores/',
    )

    # Once per city, parse stores
    def parse_city(self, city):
        city_stores = city.xpath('//div[@class="col-xs-3 cityListView products store-list-product list-view"]//address')
        city_locations = city.xpath('//script[text()[contains(.,"resultsList")]]/text()').extract_first()
        city_json = json.loads(city_locations.split('resultsList=')[-1].strip())
        if len(city_stores) > 0:
            for index, city_store in enumerate(city_stores):
                phone = city_store.xpath('.//span/text()').extract()[-1]
                city_name = city_json[index]['city']
                state = city_json[index]['state']
                zip_code = city_json[index]['zip']
                props = {
                    'addr_full': '{}, {}, {} {}'.format(city_json[index]['address'], city_name, state, zip_code),
                    'city': city_name,
                    'state': state,
                    'postcode': zip_code,
                    'phone': phone,
                    'ref': phone,
                    'website': city.urljoin(city_store.xpath('.//span/a/@href').extract_first()),
                    'name': city_store.xpath('.//span/a/text()').extract_first(),
                    'lat': float(city_json[index]['lat']),
                    'lon': float(city_json[index]['lng'])
                }
                yield GeojsonPointItem(**props)
    
    # Once per state, gets cities.
    def parse_state(self, state):
        cities = state.xpath('//div[@id="cityStateLinks"]/ul[@id="cityList"]/li/a/@href').extract()
        for city in cities:
            yield scrapy.Request(
                state.urljoin(city),
                callback=self.parse_city
            )
    
    # Initial request, gets states.
    def parse(self, response):
        states = response.xpath('//div[@id="cityStateLinks"]//ul/li/a/@href').extract()
        for state in states:
            yield scrapy.Request(
                response.urljoin(state), 
                callback=self.parse_state
            )

@@ -980,16 +980,90 @@
 %5B'zip'%5D%0A
+                web = city_store.xpath('.//span/a/@href').extract_first()%0A
         
@@ -1080,16 +1080,16 @@
 ops = %7B%0A
-
         
@@ -1374,21 +1374,19 @@
  'ref': 
-phone
+web
 ,%0A      
@@ -1427,59 +1427,11 @@
 oin(
-city_store.xpath('.//span/a/@href').extract_first()
+web
 ),%0A 
@@ -1509,17 +1509,16 @@
 _first()
-,
 %0A       
@@ -1511,32 +1511,41 @@
 irst()%0A         
+       %7D%0A
            'lat'
@@ -1543,21 +1543,54 @@
     
-'lat': float(
+     if city_json%5Bindex%5D%5B'lat'%5D != 'null' and 
 city
@@ -1608,14 +1608,23 @@
 %5D%5B'l
-at'%5D),
+ng'%5D != 'null':
 %0A   
@@ -1644,14 +1644,22 @@
     
-'lon':
+props%5B'lat'%5D =
  flo
@@ -1680,18 +1680,18 @@
 ndex%5D%5B'l
-ng
+at
 '%5D)%0A    
@@ -1694,33 +1694,81 @@
                 
-%7D
+    props%5B'lon'%5D = float(city_json%5Bindex%5D%5B'lng'%5D)
 %0A               

Check that coordinates aren't 'null' and do not use phone number as ref as it isn't unique