import scrapy
from locations.items import GeojsonPointItem
import json
from scrapy.http import XmlResponse, HtmlResponse
import re


class PlanetFitnessSpider(scrapy.Spider):
    name = "planet-fitness"
    allowed_domains = ["planetfitness.com"]

    def start_requests(self):
        sitemap_url = "https://www.planetfitness.com/sitemap.xml"
        yield scrapy.Request(url=sitemap_url, headers={"accept": "text/xml"})

    def parse(self, response):
        # wrong content-type from server, hacky solution by specifying HtmlResponse
        new_response = HtmlResponse(url=response.url, body=response.body)
        urls = [url for url in new_response.css("loc::text").extract() if "https://www.planetfitness.com/gyms/" in url]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse_gym)

    def parse_gym(self, response):
        coordinates = re.findall(r"(?<=\.setLngLat\(\[).*(?=\]\))", response.body_as_unicode())
        lat, lon = coordinates[0].split(", ") if len(coordinates) else [None, None]
        iframe = response.css("iframe[src*='https://mico.myiclubonline.com']::attr('src')")
        club_number = iframe.extract_first("=-1").split("=")[-1]
        point = {
            "lat": lat,
            "lon": lon,
            "name": "Planet Fitness " + response.css("h1.alt::text").extract_first(default="").strip(),
            "addr_full": response.css(".address-line1::text").extract_first(),
            "city": response.css(".locality::text").extract_first(),
            "state": response.css(".administrative-area::text").extract_first(),
            "postcode": response.css(".postal-code::text").extract_first(),
            "country": response.css(".country::text").extract_first(),
            "phone": response.css(".field--name-field-phone div::text").extract_first(default="").strip(),
            "website": response.url,
            "opening_hours": "24/7",
            "ref": response.url
        }

        yield GeojsonPointItem(**point)

@@ -56,70 +56,8 @@
 tem%0A
-import json%0Afrom scrapy.http import XmlResponse, HtmlResponse%0A
 impo
@@ -182,60 +182,34 @@
 m%22%5D%0A
-%0A
     
-def 
 start_
-requests(self):%0A        sitemap_url =
+urls = (%0A       
  %22ht
@@ -252,347 +252,201 @@
 xml%22
+,
 %0A    
-    yield scrapy.Request(url=sitemap_url, headers=%7B%22accept%22: %22text/xml%22%7D)%0A%0A    def parse(self, response):%0A        # wrong content-type from server, hacky solution by specifying HtmlResponse%0A        new_response = HtmlResponse(url=response.url, body=response.body)%0A        urls = %5Burl for url in new_response.css(%22loc::text%22).extract()
+)%0A%0A    def parse(self, response):%0A        response.selector.remove_namespaces()%0A        city_urls = response.xpath('//url/loc/text()').extract()%0A        for path in city_urls:%0A           
  if 
@@ -490,38 +490,18 @@
  in 
-url%5D
+path:
 %0A    
-    for url in urls:%0A
     
@@ -533,16 +533,54 @@
 est(
-url=url,
+%0A                    path,%0A                   
  cal
@@ -599,16 +599,33 @@
 arse_gym
+%0A                
 )%0A%0A    d
@@ -838,164 +838,8 @@
 ne%5D%0A
-        iframe = response.css(%22iframe%5Bsrc*='https://mico.myiclubonline.com'%5D::attr('src')%22)%0A        club_number = iframe.extract_first(%22=-1%22).split(%22=%22)%5B-1%5D
 %0A   
@@ -1529,45 +1529,8 @@
 rl,%0A
-            %22opening_hours%22: %2224/7%22,%0A
     

Use remove_namespaces to simplify spider code