from bs4 import BeautifulSoup
import requests, time, re

SCRAPE_IDS = True
REGION_SPECIFIC_SITE_URL = 'http://www.petango.com/Forms/ShelterSearchResults.aspx?z=10021&d=10&sh=0&s=4&g=All&size=All&c=All&a=All&dec=All&p=False&v=False&adoptD=False&adoptC=False&adoptO=False&sid=0&zs=True&ht=False'
NUM_REQUEST_ATTEMPTS = 5

def fetch_page_listing():

    # Get and parse page containing shelter list
    for attempt in range(NUM_REQUEST_ATTEMPTS):
        try:
            res = requests.get(REGION_SPECIFIC_SITE_URL, timeout=5)
            break
        except:
            time.sleep(5) # Wait 5 seconds before trying again

    page = BeautifulSoup(res.text,"html.parser")
    return page
    
    
if SCRAPE_IDS:
    
    regionList = fetch_page_listing()
    shelterUrls = regionList.findAll('a',{'class': 'btn-moredetails w90'})
    
    SHELTER_IDS = {}
    for shelter in shelterUrls:
        shelterPage = requests.get(shelter.attrs['href'])
        pageSoup = BeautifulSoup(shelterPage.text, "html.parser")
        listingLink = pageSoup.find('a',{'class': 'btn'}).attrs['href']
        pagePattern = re.compile(r'sh=([0-9]*)')
        pageId = pagePattern.search(listingLink)
        listingNumber = pageId.group(1).strip()
        SHELTER_IDS[shelter.attrs['href'].split('/')[-1]] = listingNumber

else:

    SHELTER_IDS = {
        '92ndStASPCA': 1950,
        'DogHabitRescue': 2903,
        'anjellicleCatsRescue': 994,
        'seanCaseyRescue': 408
        }
@@ -50,16 +50,60 @@
 me, re%0A%0A
+ZIPCODE = '10021'%0ADISTANCE_IN_MILES = '10'%0A%0A
 SCRAPE_I
@@ -112,32 +112,16 @@
  = True%0A
-REGION_SPECIFIC_
 SITE_URL
@@ -182,143 +182,8 @@
 aspx
-?z=10021&d=10&sh=0&s=4&g=All&size=All&c=All&a=All&dec=All&p=False&v=False&adoptD=False&adoptC=False&adoptO=False&sid=0&zs=True&ht=False
 '%0ANU
@@ -206,16 +206,17 @@
 TS = 5%0A%0A
+%0A
 def fetc
@@ -234,16 +234,91 @@
 ing():%0A%0A
+    params = %7B%0A        'z': ZIPCODE,%0A        'd': DISTANCE_IN_MILES%0A    %7D%0A%0A
     # Ge
@@ -454,32 +454,16 @@
 get(
-REGION_SPECIFIC_
 SITE_URL
 , ti
@@ -458,16 +458,24 @@
 SITE_URL
+, params
 , timeou
@@ -652,16 +652,21 @@
   %0A    %0A
+    %0A
 if SCRAP
@@ -1426,12 +1426,13 @@
 08%0A        %7D
+%0A

Update shelters.py