<jupyter_start><jupyter_text>Supermarket Staple Prices  Imports, Zipcodes, and Staples List  Need to install selenium and webdriver_manager  Currently have:1. Giant2. <NAME>3. Safeway4. Aldi5. Target6. Whole Foods Market7. Lidl8. Food Lion9. Amazon No online presence?:1. Publix2. Trader Joe's3. Costco<jupyter_code># imports
# generic imports
import numpy as np
import re
import time
import pandas as pd
import matplotlib.pyplot as plt

# selenium imports
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.common.exceptions import NoSuchElementException
# importing helper functions from .py file!!!
import food_helper
# Contains Arlington Zipcodes
zipcodes = np.array([20330, 22201, 22202, 22203, 22204, 22205, 22206, 22207, 22209, 22210, 22211, 22213, 22214,
                     22215, 22216, 22217, 22219, 22225, 22226, 22227, 22230, 22240, 22241, 22242, 22243, 22244,
                     22245, 22246, 22350])
# reload food_helper file
import importlib
importlib.reload(food_helper)<jupyter_output><empty_output><jupyter_text>NOT SURE WHAT IS GOING ON WITH GIANT + HARRIS TEETER??????  GIANT<jupyter_code># Giant Baskets
giant_foods = ["Giant Whole Vitamin D Milk", "Giant White Eggs Grade A", "Giant Bagels Plain Pre-Sliced 6 ct", "Giant White Bread",
               "Giant Pasta Penne Rigate", "Gala Apple", "Ground Turkey 85%", "Giant Deli Ham Honey Ham Thin Sliced",
               "Giant Sweet Cream Butter Sticks 4", "Giant Potatoes Russet", "Navel Orange", "Giant Tomatoes",
               "Ground Beef", "Giant 4 Cheese Mexican Blend", "Giant Yogurt", "Giant Cereal Shredded Wheat"]


# Napolitos?? - finish this!
giant_hispanic_foods = ["Avocado", "Giant Beets Whole", "Black Beans", "Giant 4 Cheese Mexican Blend", "Giant Corn Tortillas",
                        "Green Chiles", "Jalapeno Peppers", "Garbanzo Beans", "Masa",
                        "Giant Mixed Fruit Large", "Bell Peppers Green", "Pinto Beans", "Giant White Rice Long Grain",
                        "Pace Chunky Salsa", "Yellow Squash", "Tomatillos", "Tomatoes On The Vine"]

giant_east_af_foods = ["Curry Powder", "Dried Beans", "Premium Dates",
                       "Garbanzo Beans", "Giant Spaghetti Pasta",
                       "Giant Tomato Sauce", "Tuna"]
# had to drop fava beana + health bearley...

giant_east_eu_foods = ["Ground Beef", "Giant Beets", "Red Cabbage",
                       "Giant Lamb", "Giant Sliced Mushrooms", "Giant Pork",
                       "Giant Potatoes Russet", "Giant Wheat Flour"]
# had to drop health barley

# giant_indig_foods = []

giant_se_asia_foods = ["Baby Corn", "Coconut Milk", "Edamame", 
                       "Eggplant", "Fish Sauce", "Vermicelli",
                       "Oyster Sauce", "Rice Noodles",
                       "Jasmine Rice", "Yellow Squash", "Tofu"]
# dropped curry paste, tapioca flour, bamboo shoots
# maybe hae this check for cheap fish sauce...

giant_west_af_foods = ["Black Eye Peas", "Pinto Beans", "Cassava", "Giant Chicken Whole",
                       "Corn Flour", "Maggi", "Plantain Ripe", "Giant Potatoes Russet",
                       "Giant Tomato Paste", "Potatoes Yams"]
# add condition to skip or fill in NA for cultural items not found in some locations!
# Universal Basket for Giant
giant_df, giant_zips, giant_locs = food_helper.giant(giant_foods, zipcodes)
giant_df.head()
# Latinx Basket for Giant
giant_hispanic_df = food_helper.giant(giant_hispanic_foods, giant_zips, standard = False, has_zipcodes = True, locs = giant_locs)
giant_hispanic_df.head()
# East African Basket for Giant
giant_east_af_df = food_helper.giant(giant_east_af_foods, giant_zips, standard = False, has_zipcodes = True, locs = giant_locs)
giant_east_af_df.head()
# Eastern European Basket for Giant
giant_east_eu_df = food_helper.giant(giant_east_eu_foods, giant_zips, standard = False, has_zipcodes = True, locs = giant_locs)
giant_east_eu_df.head()
# Southeast Asian Basket for Giant
giant_se_asia_df = food_helper.giant(giant_se_asia_foods, giant_zips, standard = False, has_zipcodes = True, locs = giant_locs)
giant_se_asia_df.head()
# West African Basket for Giant
giant_west_af_df = food_helper.giant(giant_west_af_foods, giant_zips, standard = False, has_zipcodes = True, locs = giant_locs)
giant_west_af_df.head()
# Giant to csv
# giant_east_af_df.to_csv('Giant_East_Af_Food.csv')
# giant_east_eu_df.to_csv('Giant_East_Eu_Food.csv')
# giant_se_asia_df.to_csv('Giant_SE_Asia_Food.csv')
# giant_west_af_df.to_csv('Giant_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text><NAME>eter<jupyter_code># <NAME>eter Basket
ht_foods = ["Harris Teeter 3.25% Milk Fat Whole Milk", "Harris Teeter Grade A Large White Eggs", "Thomas Plain Bagels", "Harris Teeter Bread Old Fashioned",
            "<NAME> Penne", "Fresh Gala Apple", "Ground Turkey 94%", "<NAME>",
            "Harris Teeter Butter Spread", "Russet Potatoes", "Navel Oranges", "Harris Teeter Tomato Sauce",
            "Harris Teeter Ground Beef Chuck", "<NAME>eter Shredded Cheese", "Harris Teeter Nonfat Plain Yogurt", "<NAME> Cereal"]

ht_hispanic_foods = ["Hass Avocado", "Fresh Beets", "Harris Teeter Black Beans", "Shredded Mexican Cheese Blend", "<NAME>eter Corn Tortillas",
                     "Harris Teeter Green Chiles", "Jalapeno Peppers", "Harris Teeter Garbanzo Beans",
                     "Mixed Fruit", "Bell Peppers Green", "Harris Teeter Pinto Beans", "Harris Teeter Long Grain White Rice",
                     "<NAME>", "Zucchini Squash", "Tomatillos", "Roma Tomato"]

ht_east_af_foods = ["Pearled Barley", "Curry Powder", "Dried Beans", "Sunsweet Dates",
                    "<NAME> Garbanzo Beans", "Lentil Beans", "<NAME>aghetti",
                    "<NAME> Tomato Sauce", "Tuna"]

ht_east_eu_foods = ["Pearled Barley", "<NAME> Ground Beef Chuck", "Fresh Beets",
                    "Buckwheat", "Fresh Cabbage Red", "Lamb",
                    "<NAME> Whole White Mushrooms", "Ground Pork Lean", "Russet Potatoes",
                    "Veal", "Wheat Flour"]

# ht_indig_foods = []

ht_se_asia_foods = ["HT Bamboo Shoots", "Baby Corn", "HT Coconut Milk", "Thai Curry Paste",
                    "Edamame", "Eggplant", "HT Fish Sauce", "Vermicelli",
                    "Oyster Sauce", "Rice Flour", "Rice Noodles", "Jasmine Rice Mahatma Thai",
                    "Fresh Squash", "Tapioca Flour", "Tofu"]

ht_west_af_foods = ["<NAME> Dry Blackeye Peas", "<NAME> Dry Pinto Beans", "Chicken Whole",
                    "Fresh Plantain", "Russet Potatoes", "<NAME> Tomato Paste"]
# Universal Basket for <NAME>eter
ht_df, ht_zips, ht_locs = food_helper.ht(ht_foods, zipcodes)
ht_df.head()
# Latinx Basket for Harris Teeter
ht_hispanic_df = food_helper.ht(ht_hispanic_foods, ht_zips, standard = False, has_zipcodes = True, locs = ht_locs)
ht_hispanic_df.head()
# East African Basket for Harris Teeter
ht_east_af_df = food_helper.ht(ht_east_af_foods, ht_zips, standard = False, has_zipcodes = True, locs = ht_locs)
ht_east_af_df.head()
# Eastern European Basket for Harris Teeter
ht_east_eu_df = food_helper.ht(ht_east_eu_foods, ht_zips, standard = False, has_zipcodes = True, locs = ht_locs)
ht_east_eu_df.head()
# Southeast Asian Basket for Harris Teeter
ht_se_asia_df = food_helper.ht(ht_se_asia_foods, ht_zips, standard = False, has_zipcodes = True, locs = ht_locs)
ht_se_asia_df.head()
# West African Basket for Harris Teeter
ht_west_af_df = food_helper.ht(ht_west_af_foods, ht_zips, standard = False, has_zipcodes = True, locs = ht_locs)
ht_west_af_df.head()
# Harris Teeter to csv
# ht_east_af_df.to_csv('HT_East_Af_Food.csv')
# ht_east_eu_df.to_csv('HT_East_Eu_Food.csv')
# ht_se_asia_df.to_csv('HT_SE_Asia_Food.csv')
# ht_west_af_df.to_csv('HT_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text>Safeway<jupyter_code># Safeway Basket
sw_foods = ["Value Corner Whole Milk - 1 Gallon", "Large Eggs - 12 Count", "Plain Bagels", "signature Select Bread White",
            "Signature Select Pasta 16 oz", "Gala Apple", "Signature Farms Ground Turkey", "Signature Select Ham Cooked 95%",
            "Value Corner Butter", "Russet Potatoes 5", "Navel Oranges", "Signature Select Tomato Sauce 8",
            "Ground Beef 80% 1.25", "Shredded Cheese 8", "Yogurt Lucerne 32", "Signature Select Cereal Corn Flakes"]

sw_hispanic_foods = ["Hass Avocado", "Beets - 1 Bunch", "Signature Select Black Beans", "Mexican Cheese Blend 8", "Signature Select Corn Tortillas",
                     "Signature Select Green Chiles", "Jalapeno Peppers", "Signature Select Garbanzo Beans", "Masa",
                     "Giant Mixed Fruit Large", "Bell Peppers Green", "Signature Select Pinto Beans", "White Rice 5lb",
                     "Signature Select Salsa 16", "Green Squash", "Tomatillos", "Tomatoes On The Vine"]
# Mixed fruit?

sw_east_af_foods = ["Barley", "Simply Curry Powder", "Dried Beans", "Sunsweet Dates",
                    "Fava Beans", "Garbanzo Beans", "Lentil Beans", "Signature Spaghetti",
                    "Signature Tomato Sauce", "Select Tuna Chunk"]

sw_east_eu_foods = ["Barley", "Ground Beef 80% 1.25", "Beets 1 Bunch",
                    "Red Cabbage", "Lamb Shoulder", "Sliced Mushroom",
                    "Ground Pork", "Russet Potatoes", "Wheat Flour"]
# had to drop kasha
# sort beets by price

# sw_indig_foods = []

sw_se_asia_foods = ["Light Coconut Milk", "Signature Curry Paste", "Edamame",
                    "Eggplant", "Fish Sauce", "Oyster Sauce", "Rice Flour", "Rice Noodles",
                    "Jasmine Rice", "Squash", "Tofu"]
# had to drop bamboo shoots - was getting toothbrush or something?

sw_west_af_foods = ["Black eye Peas", "Dry Pinto Beans", "Chicken Whole", "Corn Flour",
                    "Plantain", "Russet Potatoes", "Tomato Paste", "Yams"]
# Universal Basket for Safeway
sw_df, sw_zips, sw_locs = food_helper.sw(sw_foods, zipcodes)
sw_df.head()
# Latinx Basket for Safeway
sw_hispanic_df = food_helper.sw(sw_hispanic_foods, sw_zips, standard = False, has_zipcodes = True, locs = sw_locs)
sw_hispanic_df.head()
# East African Basket for Safeway
sw_east_af_df = food_helper.sw(sw_east_af_foods, sw_zips, standard = False, has_zipcodes = True, locs = sw_locs)
sw_east_af_df.head()
# Eastern European Basket for Safeway
sw_east_eu_df = food_helper.sw(sw_east_eu_foods, sw_zips, standard = False, has_zipcodes = True, locs = sw_locs)
sw_east_eu_df.head()
# Southeast Asian Basket for Safeway
sw_se_asia_df = food_helper.sw(sw_se_asia_foods, sw_zips, standard = False, has_zipcodes = True, locs = sw_locs)
sw_se_asia_df.head()
# West African Basket for Safeway
sw_west_af_df = food_helper.sw(sw_west_af_foods, sw_zips, standard = False, has_zipcodes = True, locs = sw_locs)
sw_west_af_df.head()
# Safeway to csv
# sw_east_af_df.to_csv('SW_East_Af_Food.csv')
# sw_east_eu_df.to_csv('SW_East_Eu_Food.csv')
# sw_se_asia_df.to_csv('SW_SE_Asia_Food.csv')
# sw_west_af_df.to_csv('SW_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text>Aldi<jupyter_code># Aldi Basket
aldi_foods = ["Whole Milk", "Large Eggs A Goldhen Eggs", "L'oven Everything", "White Bread",
              "Penne Rigate", "Gala Apple", "Turkey Ground Lean 93", "Deli Sliced Ham",
              "Countryside Salted Butter", "Bag Potatoes", "Oranges Bag", "Happy Harvest Tomato Sauce",
              "Lean Beef Chub", "Happy Cheese Cubes", "Friendly Farms Nonfat Yogurt Light Plain", "Millville Frosted Shredded"]

aldi_hispanic_foods = ["Large Avocado", "Dakota Black Beans", "Mexican Cheese", "Corn Tortillas",
                       "Green Chiles", "Jalapeno Peppers", "Garbanzo Beans", "Masa",
                       "Mixed Fruit", "Bell Peppers Green", "Pinto Beans", "White Rice",
                       "Casa Medium Salsa", "Yellow Squash", "Tomatillos", "Roma Tomatoes"]
## need to fix the food items in this list.......

# no beets
# no tomatilos???

aldi_east_af_foods = ["Pueblo Pinto Beans", "Garbanzo Beans", "Spaghetti",
                      "Tomato Sauce", "Canned Tuna"]

aldi_east_eu_foods = ["Lean Beef Chub", "Cabbage", "Lamb", "Sliced Mushrooms",
                      "Pork", "Bag Potatoes"]
# sort beets by price

# aldi_indig_foods = []

aldi_se_asia_foods = ["Coconut Milk", "Eggplant", "Parboiled Rice",
                      "Yellow Squash", "Tofu"]
# number 2 option here for coconut milk!

aldi_west_af_foods = ["Dry Pinto Beans", "Chicken Whole", "Corn Flour", "Plantain",
                      "Bag Potatoes", "Tomato Paste", "Yams"]
# Universal Basket for Aldi
aldi_df, aldi_zips, aldi_locs = food_helper.aldi(aldi_foods, zipcodes)
aldi_df.head()
# Latinx Basket for Aldi
aldi_hispanic_df = food_helper.aldi(aldi_hispanic_foods, aldi_zips, standard = False, has_zipcodes = True, locs = aldi_locs)
aldi_hispanic_df.head()
# East African Basket for Aldi
aldi_east_af_df = food_helper.aldi(aldi_east_af_foods, aldi_zips, standard = False, has_zipcodes = True, locs = aldi_locs)
aldi_east_af_df.head()
# Eastern European Basket for Aldi
aldi_east_eu_df = food_helper.aldi(aldi_east_eu_foods, aldi_zips, standard = False, has_zipcodes = True, locs = aldi_locs)
aldi_east_eu_df.head() 
# Southeast Asian Basket for Aldi
aldi_se_asia_df = food_helper.aldi(aldi_se_asia_foods, aldi_zips, standard = False, has_zipcodes = True, locs = aldi_locs)
aldi_se_asia_df.head()
# West African Basket for Aldi
aldi_west_af_df = food_helper.aldi(aldi_west_af_foods, aldi_zips, standard = False, has_zipcodes = True, locs = aldi_locs)
aldi_west_af_df.head()
# Aldi to csv
# aldi_east_af_df.to_csv('Aldi_East_Af_Food.csv')
# aldi_east_eu_df.to_csv('Aldi_East_Eu_Food.csv')
# aldi_se_asia_df.to_csv('Aldi_SE_Asia_Food.csv')
# aldi_west_af_df.to_csv('Aldi_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text>Target<jupyter_code># Target Basket
target_foods = ["Vitamin D Whole Milk - 1 gal", "Grade A Large Eggs 12", "Plain Bagel 6", "Bread Market Pantry",
                "Penne Rigate 16", "Gala Apple Each", "Ground Turkey", "Sliced Ham",
                "Butter", "Russet Potatoes", "Navel Orange", "Tomato Sauce 8oz - Good & Gather",
                "Ground Beef 1lb 80", "Shredded Cheese", "Yoplait Yogurt 32oz", "Market Pantry Cereal"]

target_hispanic_foods = ["Avocado", "Good and Gather Black Beans", "Mexican Cheese", "Corn Tortillas",
                         "Green Chiles", "Jalapeno Peppers", "Garbanzo Beans", "Masa",
                         "Fruit Cocktail Market", "Bell Peppers", "Pinto Beans", "White Rice 5",
                         "Medium Salsa 24", "Squash", "Good and Gather Salsa Verde", "Roma Tomatoes"]
# SOME HAVE TOMATILLOS, OTHERS TOMATILLO SALSA, AND OTHERS SALSA VERDE...

target_east_af_foods = ["Barley", "Badia Curry Powder", "Dried Beans", "Dried Dates",
                        "Garbanzo Beans", "Faraon Lentil Beans", "Good And Gather Spaghetti",
                        "Tomato Sauce 8oz", "Good And Gather Canned Tuna"]
# sort tomato sauce by price

target_east_eu_foods = ["Barley", "Ground Beef 80%", "Good and Gather Sliced Beets", "Cabbage",
                        "Sliced White Mushrooms", "Russet Potatoes", "Wheat Flour"]
# sort beets by price

# target_indig_foods = []

target_se_asia_foods = ["Baby Corn", "Coconut Milk", "Patak Curry Paste", "Edamame",
                        "Three Crabs Fish Sauce", "Thai Vermicelli", "Rice Noodles", "Jasmine Rice"]
# number 2 option here for coconut milk!

target_west_af_foods = ["Dry Black Eye Peas", "Dry Pinto Beans", "Corn Flour", "Maggi Cubes Chicken",
                        "Platano", "Russet Potatoes", "Tomato Paste", "Yams"]
# unsure on chicken...
# make alteration for platano!!1
# Universal Basket for Target
target_df, target_zips, target_locs = food_helper.target(target_foods, zipcodes)
target_df.head()
# Latinx Basket for Target
target_hispanic_df = food_helper.target(target_hispanic_foods, target_zips, standard = False, has_zipcodes = True, locs = target_locs)
target_hispanic_df.head()
# East African Basket for Target
target_east_af_df = food_helper.target(target_east_af_foods, target_zips, standard = False, has_zipcodes = True, locs = target_locs)
target_east_af_df.head()
# Eastern European Basket for Target
target_east_eu_df = food_helper.target(target_east_eu_foods, target_zips, standard = False, has_zipcodes = True, locs = target_locs)
target_east_eu_df.head()
# Southeast Asian Basket for Target
target_se_asia_df = food_helper.target(target_se_asia_foods, target_zips, standard = False, has_zipcodes = True, locs = target_locs)
target_se_asia_df.head()
# West African Basket for Target
target_west_af_df = food_helper.target(target_west_af_foods, target_zips, standard = False, has_zipcodes = True, locs = target_locs)
target_west_af_df.head()
# Target to csv
# target_east_af_df.to_csv('Target_East_Af_Food.csv')
# target_east_eu_df.to_csv('Target_East_Eu_Food.csv')
# target_se_asia_df.to_csv('Target_SE_Asia_Food.csv')
# target_west_af_df.to_csv('Target_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text>Whole Foods<jupyter_code># Whole Foods Basket
wfm_foods = ["Whole Milk, 1 Gallon", "Large Eggs", "Plain Bagels", "Sandwich Bread Whole Wheat",
             "Pasta Penne Rigate", "Fuji Apple", "Ground Turkey Thigh", "Sliced Ham Value Pack",
             "Butter, Unsalted 4 Quarters", "Organic Russet Potato", "Navel Orange", "Tomato Sauce",
             "Ground Beef 80%", "Shredded Mexican Blend", "Organic Yogurt Whole Milk", "Frosted Wheat Squares"]

wfm_hispanic_foods = ["Large Hass Avocado", "Shelf-Stable Black Beans", "Shreddedd Mexican Blend", "Organic Tortillas",
                      "Large Green Chiles", "Jalapeno Peppers", "Shelf-Stable Garbanzo Salt",
                      "Green Bell Pepper", "Shelf-Stable Pinto Salt", "White Rice 80",
                      "Medium Salsa Batch", "Squash", "Tomatillo", "Roma Tomato"]
# No Masa
# no mixed fruit?

wfm_east_af_foods = ["Barley", "Curry Powder", "Dried Fruit Dates",
                     "Salt Stable Garbanzo", "Red Lentils", "Pasta Spaghetti",
                     "Teff", "Tomato Sauce", "Canned Tuna 5oz Everday"]

wfm_east_eu_foods = ["Barley", "Ground Beef 80%", "Red Beet Bunch", "Cabbage",
                     "Sliced White Mushrooms", "Pork", "Russet Potatoes", "Wheat Flour"]

# wfm_indig_foods = []

wfm_se_asia_foods = ["Bamboo Shoots", "Baby Corn", "Vanilla Coconut Milk", "Thai Curry Paste",
                     "Fish Sauce", "Vermicelli", "Rice Flour", "Rice Noodles",
                     "Thai Jasmine Rice", "Yellow Squash", "Tofu"]

wfm_west_af_foods = ["Black Eyed Peas", "Shelf Salt Pinto", "Whole Chicken", "Plantain",
                     "Russet Potatoes", "Tomato Paste"]<jupyter_output><empty_output><jupyter_text>Probably want to change below: visit<jupyter_code># Universal Basket for Whole Foods
wfm_df, wfm_zips, wfm_locs = food_helper.wfm(wfm_foods, zipcodes)
wfm_df.head()
# Latinx Basket for Whole Foods
wfm_hispanic_df = food_helper.wfm(wfm_hispanic_foods, wfm_zips, standard = False, has_zipcodes = True, locs = wfm_locs)
wfm_hispanic_df.head()
# East African Basket for Whole Foods
wfm_east_af_df = food_helper.wfm(wfm_east_af_foods, wfm_zips, standard = False, has_zipcodes = True, locs = wfm_locs)
wfm_east_af_df.head()
# Eastern European Basket for Whole Foods
wfm_east_eu_df = food_helper.wfm(wfm_east_eu_foods, wfm_zips, standard = False, east_euro = True, has_zipcodes = True, locs = wfm_locs)
wfm_east_eu_df.head()
# Southeast Asian Basket for Whole Foods
wfm_se_asia_df = food_helper.wfm(wfm_se_asia_foods, wfm_zips, standard = False, has_zipcodes = True, locs = wfm_locs)
wfm_se_asia_df.head()
# East African Basket for Whole Foods
wfm_west_af_df = food_helper.wfm(wfm_west_af_foods, wfm_zips, standard = False, west_af = True, has_zipcodes = True, locs = wfm_locs)
wfm_west_af_df.head()
# Whole Foods to csv
# wfm_east_af_df.to_csv('WFM_East_Af_Food.csv')
# wfm_east_eu_df.to_csv('WFM_East_Eu_Food.csv')
# wfm_se_asia_df.to_csv('WFM_SE_Asia_Food.csv')
# wfm_west_af_df.to_csv('WFM_West_Af_Food.csv')<jupyter_output><empty_output><jupyter_text>Lidl<jupyter_code># Lidl Baskets
# lidl
lidl_foods = ["Whole Milk", "12 Large Eggs", "Bagels", "Wheat Bread",
              "Penne Pasta Rigate", "Gala Apples", "Ground Turkey", "Sliced Ham",
              "Salted Butter", "Russet Potatoes", "Oranges", "Roma Tomatoes",
              "80% Ground Beef", "Mexican Cheese", "Yogurt", "Frosted Flakes Cereal"]
# second option with whole milk

lidl_latinx_basket = ["Avocado", "Beets", "Black Beans", "Mexican Cheese",
                      "Small Corn Tortilla", "Green Bell Peppers, 3 Count",
                      "Pinto Beans", "Long Grain Enriched White Rice",
                      "Medium Salsa", "Squash", "Tomatillos", "Roma Tomatoes"]
# coocked beats??
# are we interested in canned or dry beans??

lidl_east_af_basket = ["Pinto Beans", "Dried Dates", "Garbanzo Beans", "Lentils", "Basmati Rice",
                       "Spaghetti Pasta", "Tomato Sauce", "Tuna"]

lidl_east_eu_basket = ["80% Ground Beef", "Beets", "Green Cabbage", "Ground Lamb", "Lentils", 
                       "Mushrooms", "Pork", "Russet Potatoes", "Wheat Flour"]

lidl_se_asia_basket = ["Goya Coconut Milk", "Edamame", "Eggplant",
                       "Taste of Thai Rice Noodles", "Jasmine Rice", "Squash"]

lidl_west_af_basket = ["Black Eyed Peas", "Pinto Beans", "Farm Chicken Drumsticks", "Chicken Bouillon Cubes",
                       "Russet Potatoes", "Tomato Paste", "Red Yams", "Yuca"]
# what part of chicken...

lidl_baskets = [lidl_foods, lidl_latinx_basket, lidl_east_af_basket,
              lidl_east_eu_basket, lidl_se_asia_basket, lidl_west_af_basket]

# Lidl Drivers for all groups and to csv!
lidl_names = np.array(['Lidl_Universal_Food.csv', 'Lidl_Latinx_Food.csv', 'Lidl_East_Af_Food.csv',
                  'Lidl_East_Eu_Food.csv', 'Lidl_SE_Asia_Food.csv', 'Lidl_West_Af_Food.csv'])

food_helper.lidl(lidl_names, zipcodes, [], 2, 20, True, lidl_foods, lidl_latinx_basket,
                 lidl_east_af_basket, lidl_east_eu_basket, lidl_se_asia_basket, lidl_west_af_basket)<jupyter_output>====== WebDriver manager ======
Current google-chrome version is 97.0.4692
Get LATEST driver version for 97.0.4692
Driver [/Users/zacharyalerte/.wdm/drivers/chromedriver/mac64/97.0.4692.71/chromedriver] found in cache<jupyter_text>Putting DataFrames together<jupyter_code># # TO CSV!
# # Universal
# giant_df.to_csv('Giant_Universal_Food.csv')
# ht_df.to_csv('HT_Universal_Food.csv')
# sw_df.to_csv('SW_Universal_Food.csv')
# aldi_df.to_csv('Aldi_Universal_Food.csv')
# target_df.to_csv('Target_Universal_Food.csv')
# wfm_df.to_csv('WFM_Universal_Food.csv')

# # Latinx
# giant_hispanic_df.to_csv('Giant_Latinx_Food.csv')
# ht_hispanic_df.to_csv('HT_Latinx_Food.csv')
# sw_hispanic_df.to_csv('SW_Latinx_Food.csv')
# aldi_hispanic_df.to_csv('Aldi_Latinx_Food.csv')
# target_hispanic_df.to_csv('Target_Latinx_Food.csv')
# wfm_hispanic_df.to_csv('WFM_Latinx_Food.csv')

# # East African
# giant_east_af_df.to_csv('Giant_East_Af_Food.csv')
# ht_east_af_df.to_csv('HT_East_Af_Food.csv')
# sw_east_af_df.to_csv('SW_East_Af_Food.csv')
# aldi_east_af_df.to_csv('Aldi_East_Af_Food.csv')
# target_east_af_df.to_csv('Target_East_Af_Food.csv')
# wfm_east_af_df.to_csv('WFM_East_Af_Food.csv')

# # Eastern European
# giant_east_eu_df.to_csv('Giant_East_Eu_Food.csv')
# ht_east_eu_df.to_csv('HT_East_Eu_Food.csv')
# sw_east_eu_df.to_csv('SW_East_Eu_Food.csv')
# aldi_east_eu_df.to_csv('Aldi_East_Eu_Food.csv')
# target_east_eu_df.to_csv('Target_East_Eu_Food.csv')
# wfm_east_eu_df.to_csv('WFM_East_Eu_Food.csv')

# # Indigenous/Native
# # giant_df.to_csv('Giant_Universal_Food.csv')
# # ht_df.to_csv('HT_Universal_Food.csv')
# # sw_df.to_csv('SW_Universal_Food.csv')
# # aldi_df.to_csv('Aldi_Universal_Food.csv')
# # target_df.to_csv('Target_Universal_Food.csv')
# # wfm_df.to_csv('WFM_Universal_Food.csv')

# # Southeast Asian
# giant_se_asia_df.to_csv('Giant_SE_Asia_Food.csv')
# ht_se_asia_df.to_csv('HT_SE_Asia_Food.csv')
# sw_se_asia_df.to_csv('SW_SE_Asia_Food.csv')
# aldi_se_asia_df.to_csv('Aldi_SE_Asia_Food.csv')
# target_se_asia_df.to_csv('Target_SE_Asia_Food.csv')
# wfm_se_asia_df.to_csv('WFM_SE_Asia_Food.csv')

# # West African
# giant_west_af_df.to_csv('Giant_West_Af_Food.csv')
# ht_west_af_df.to_csv('HT_West_Af_Food.csv')
# sw_west_af_df.to_csv('SW_West_Af_Food.csv')
# aldi_west_af_df.to_csv('Aldi_West_Af_Food.csv')
# target_west_af_df.to_csv('Target_West_Af_Food.csv')
# wfm_west_af_df.to_csv('WFM_West_Af_Food.csv')
# # Put data together by basket across stores and into CSVs!
# # Universal
# universal_df = pd.concat([giant_df, ht_df, sw_df, aldi_df, target_df, wfm_df], ignore_index=True)
# display(universal_df.head())
# universal_df.to_csv('Universal_Food.csv')

# # Latinx
# latinx_df = pd.concat([giant_hispanic_df, ht_hispanic_df, sw_hispanic_df, aldi_hispanic_df, target_hispanic_df, wfm_hispanic_df], ignore_index=True)
# display(latinx_df.head())
# latinx_df.to_csv('Latinx_Food.csv')<jupyter_output><empty_output><jupyter_text>Load dataframes for inspection<jupyter_code># LOAD DATAFRAMES
# Universal
giant_df = pd.read_csv('Giant_Universal_Food.csv')
ht_df = pd.read_csv('HT_Universal_Food.csv')
sw_df = pd.read_csv('SW_Universal_Food.csv')
aldi_df = pd.read_csv('Aldi_Universal_Food.csv')
target_df = pd.read_csv('Target_Universal_Food.csv')
wfm_df = pd.read_csv('WFM_Universal_Food.csv')
lidl_df = pd.read_csv('Lidl_Universal_Food.csv', index_col = 0)

# Latinx
giant_hispanic_df = pd.read_csv('Giant_Latinx_Food.csv')
ht_hispanic_df = pd.read_csv('HT_Latinx_Food.csv')
sw_hispanic_df = pd.read_csv('SW_Latinx_Food.csv')
aldi_hispanic_df = pd.read_csv('Aldi_Latinx_Food.csv')
target_hispanic_df = pd.read_csv('Target_Latinx_Food.csv')
wfm_hispanic_df = pd.read_csv('WFM_Latinx_Food.csv')
lidl_hispanic_df = pd.read_csv('Lidl_Latinx_Food.csv', index_col = 0)

# East African
giant_east_af_df = pd.read_csv('Giant_East_Af_Food.csv')
ht_east_af_df = pd.read_csv('HT_East_Af_Food.csv')
sw_east_af_df = pd.read_csv('SW_East_Af_Food.csv')
aldi_east_af_df = pd.read_csv('Aldi_East_Af_Food.csv')
target_east_af_df = pd.read_csv('Target_East_Af_Food.csv')
wfm_east_af_df = pd.read_csv('WFM_East_Af_Food.csv')
lidl_east_af_df = pd.read_csv('Lidl_East_Af_Food.csv', index_col = 0)

# Eastern European
giant_east_eu_df = pd.read_csv('Giant_East_Eu_Food.csv')
ht_east_eu_df = pd.read_csv('HT_East_Eu_Food.csv')
sw_east_eu_df = pd.read_csv('SW_East_Eu_Food.csv')
aldi_east_eu_df = pd.read_csv('Aldi_East_Eu_Food.csv')
target_east_eu_df = pd.read_csv('Target_East_Eu_Food.csv')
wfm_east_eu_df = pd.read_csv('WFM_East_Eu_Food.csv')
lidl_east_eu_df = pd.read_csv('Lidl_East_Eu_Food.csv', index_col = 0)

# Indigenous/Native
# giant_df = pd.read_csv('Giant_Universal_Food.csv')
# ht_df = pd.read_csv('HT_Universal_Food.csv')
# sw_df = pd.read_csv('SW_Universal_Food.csv')
# aldi_df = pd.read_csv('Aldi_Universal_Food.csv')
# target_df = pd.read_csv('Target_Universal_Food.csv')
# wfm_df = pd.read_csv('WFM_Universal_Food.csv')

# Southeast Asian
giant_se_asia_df = pd.read_csv('Giant_SE_Asia_Food.csv')
ht_se_asia_df = pd.read_csv('HT_SE_Asia_Food.csv')
sw_se_asia_df = pd.read_csv('SW_SE_Asia_Food.csv')
aldi_se_asia_df = pd.read_csv('Aldi_SE_Asia_Food.csv')
target_se_asia_df = pd.read_csv('Target_SE_Asia_Food.csv')
wfm_se_asia_df = pd.read_csv('WFM_SE_Asia_Food.csv')
lidl_se_asia_df = pd.read_csv('Lidl_SE_Asia_Food.csv', index_col = 0)

# West African
giant_west_af_df = pd.read_csv('Giant_West_Af_Food.csv')
ht_west_af_df = pd.read_csv('HT_West_Af_Food.csv')
sw_west_af_df = pd.read_csv('SW_West_Af_Food.csv')
aldi_west_af_df = pd.read_csv('Aldi_West_Af_Food.csv')
target_west_af_df = pd.read_csv('Target_West_Af_Food.csv')
wfm_west_af_df = pd.read_csv('WFM_West_Af_Food.csv')
lidl_west_af_df = pd.read_csv('Lidl_West_AF_Food.csv', index_col = 0)

# COMPILE LARGER DATAFRAMES
# Universal
universal_df = pd.concat([giant_df, ht_df, sw_df, aldi_df,
                          target_df, wfm_df, lidl_df], ignore_index = True)
universal_df = universal_df.loc[:, ~universal_df.columns.str.contains('^Unnamed')]
display(universal_df.head())
universal_df.to_csv('Universal_Food.csv')

# Latinx
latinx_df = pd.concat([giant_hispanic_df, ht_hispanic_df, sw_hispanic_df, aldi_hispanic_df,
                       target_hispanic_df, wfm_hispanic_df, lidl_hispanic_df], ignore_index = True)
latinx_df = latinx_df.loc[:, ~latinx_df.columns.str.contains('^Unnamed')]
display(latinx_df.head())
latinx_df.to_csv('Latinx_Food.csv')

# East African
east_af_df = pd.concat([giant_east_af_df, ht_east_af_df, sw_east_af_df, aldi_east_af_df,
                        target_east_af_df, wfm_east_af_df, lidl_east_af_df], ignore_index = True)
east_af_df = east_af_df.loc[:, ~east_af_df.columns.str.contains('^Unnamed')]
display(east_af_df.head())
east_af_df.to_csv('East_Af_Food.csv')

# Eastern European
east_eu_df = pd.concat([giant_east_eu_df, ht_east_eu_df, sw_east_eu_df, aldi_east_eu_df,
                        target_east_eu_df, wfm_east_eu_df, lidl_east_eu_df], ignore_index = True)
east_eu_df = east_eu_df.loc[:, ~east_eu_df.columns.str.contains('^Unnamed')]
display(east_eu_df.head())
east_eu_df.to_csv('East_Eu_Food.csv')

# Southeast Asian
se_asia_df = pd.concat([giant_se_asia_df, ht_se_asia_df, sw_se_asia_df, aldi_se_asia_df,
                        target_se_asia_df, wfm_se_asia_df, lidl_se_asia_df], ignore_index = True)
se_asia_df = se_asia_df.loc[:, ~se_asia_df.columns.str.contains('^Unnamed')]
display(se_asia_df.head())
se_asia_df.to_csv('SE_Asia_Food.csv')

# West African
west_af_df = pd.concat([giant_west_af_df, ht_west_af_df, sw_west_af_df, aldi_west_af_df,
                        target_west_af_df, wfm_west_af_df, lidl_west_af_df], ignore_index = True)
west_af_df = west_af_df.loc[:, ~west_af_df.columns.str.contains('^Unnamed')]
display(west_af_df.head())
west_af_df.to_csv('West_Af_Food.csv')
universal_df = pd.read_csv('Universal_Food.csv', index_col = 0)
latinx_df = pd.read_csv('Latinx_Food.csv', index_col = 0)
east_af_df = pd.read_csv('East_Af_Food.csv', index_col = 0)
east_eu_df = pd.read_csv('East_Eu_Food.csv', index_col = 0)
se_asia_df = pd.read_csv('SE_Asia_Food.csv', index_col = 0)
west_af_df = pd.read_csv('West_Af_Food.csv', index_col = 0)


universal_df
universal_df[::16]
latinx_df
# Instacart scrape...
foods = ["Potatoes", "Oranges", "Tomatoes", "Apples", "Ground Turkey", "Chicken Nuggets", "Ground Beef", 
         "Chicken Eggs", "Shredded Cheese", 'Whole Milk', "Butter", "Yogurt", "Multigrain Bread", "Pasta",
         "Bagels", "Cereal"]


ic_driver = webdriver.Chrome(ChromeDriverManager().install())
ic_driver.get("https://www.instacart.com/store/")
ic_driver.maximize_window()

sleep_time = 2
driver_wait = 20
wait = WebDriverWait(ic_driver, driver_wait)
log_in = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'css-1cfqs9t')))
log_in.click()

#                                                           /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[1]/div/div[1]/input
email = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[1]/div/div[1]/input')))
email.send_keys('<EMAIL>')

#                                                              /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input
password = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input')))
password.send_keys('<PASSWORD>')

#                                                                  /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[4]/button
click_log_in = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[4]/button')))
click_log_in.click()


go_store = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div[1]/div[2]/div/div[4]/ul/li[1]/a')))
go_store.click()

go_top = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/header/div/div/div[2]/div[4]/div[2]/div/div/div[1]/div[4]/button')))
go_top.click()

info = []
for food in foods:
    ic_driver.get('https://www.instacart.com/store/publix/search_v3/{}'.format(food))
    element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div/div/div/div/div[1]/div/div[2]/div/div/div[4]/ul[1]/li[1]/div/div/div/div/a/div[3]')))
    info.append(element.text)
# Instacart scrape...
foods = ["Potatoes", "Oranges", "Tomatoes", "Apples", "Ground Turkey", "Chicken Nuggets", "Ground Beef", 
         "Chicken Eggs", "Shredded Cheese", 'Whole Milk', "Butter", "Yogurt", "Multigrain Bread", "Pasta",
         "Bagels", "Cereal"]


ic_driver = webdriver.Chrome(ChromeDriverManager().install())
ic_driver.get("https://www.instacart.com/store/")
ic_driver.maximize_window()

sleep_time = 2
driver_wait = 20
wait = WebDriverWait(ic_driver, driver_wait)
log_in = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'css-s55bq')))
log_in.click()#<button class="css-s55bq"><span class="css-utfnc">Log in</span></button>

#                                                           /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[1]/div/div[1]/input
email = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[1]/div/div[1]/input')))
email.send_keys('<EMAIL>')

#                                                              /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input
password = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input')))
password.send_keys('<PASSWORD>')

#                                                                  /html/body/div[4]/div/div/div/div/div[2]/div[1]/form/div[4]/button
click_log_in = wait.until(EC.element_to_be_clickable((By.XPATH, '//div/div/div/div/div[2]/div[1]/form/div[4]/button')))
click_log_in.click()

# ERROR HERE
# <a class="css-d20dbd" href="/store/publix/storefront"><span class="css-12thh49"><span class="css-ks8hbe-StoreCompactCard"><span class="css-jewpb8-StoreCompactCard"></span><img src="https://d2d8wwwkmhfcva.cloudfront.net/72x/d2lnr5mha7bycj.cloudfront.net/warehouse/logo/57/29520839-7042-45a0-af82-54f973b4abe5.png" srcset="https://d2d8wwwkmhfcva.cloudfront.net/108x/d2lnr5mha7bycj.cloudfront.net/warehouse/logo/57/29520839-7042-45a0-af82-54f973b4abe5.png 1.5x, https://d2d8wwwkmhfcva.cloudfront.net/144x/d2lnr5mha7bycj.cloudfront.net/warehouse/logo/57/29520839-7042-45a0-af82-54f973b4abe5.png 2x" alt="Publix" width="60" height="60" loading="lazy" role="presentation" class="css-1qsdlnh-StoreCompactCard"></span><span class="css-1im00gz-StoreCompactCard"><span><span class="css-lf0nfd">Publix</span></span><span class="css-2jq6rl">Deli • Groceries • Organic</span><span class="css-e8mvzb"><span class="css-pjl6ah"><span class="css-i5e22u">Pickup 37.9mi</span></span><span class="css-pjl6ah"><span class="css-i5e22u">Accepts EBT</span></span><span class="css-pjl6ah"><span class="css-i5e22u">New</span></span></span></span></span></a>
go_store = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'css-d20dbd')))
go_store.click()

# go_top = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/header/div/div/div[2]/div[4]/div[2]/div/div/div[1]/div[4]/button')))
# go_top.click()

info = []
for food in foods:
    ic_driver.get('https://www.instacart.com/store/publix/search_v3/{}'.format(food))
    element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div/div/div/div/div[1]/div/div[2]/div/div/div[4]/ul[1]/li[1]/div/div/div/div/a/div[3]')))
    info.append(element.text)<jupyter_output>====== WebDriver manager ======
Current google-chrome version is 96.0.4664
Get LATEST driver version for 96.0.4664
Driver [/Users/zacharyalerte/.wdm/drivers/chromedriver/mac64/96.0.4664.45/chromedriver] found in cache<jupyter_text>Looking to add other stores in the DMV (Virginia specifically for VDH)  Walmart<jupyter_code># lists of food items for Walmart
walmart_foods = ["Whole Milk", "Great Value Eggs 12 Count", "Great Value Bagels", "Great Value Bread",
                 "Great Value Pasta", "Apples", "Ground Turkey", "Great Value Sliced Ham",
                 "Butter", "Potatoes", "Oranges", "Tomatoes",
                 "Ground Beef", "Shredded Cheese", "Great Value Yogurt", "Shredded Great Value Cereal 24"]

walmart_latinx_basket = ["Avocado", "Beets", "Great Value Black Beans, 16oz", "Great Value Shredded Cheese",
                         "Great Value Corn Tortilla", "Great Value Green Chiles", "Great Value Jalapenos",
                         "Great Value Corn Flour", "Green Bell Pepper", "Great Value Pinto Beans, 32oz",
                         "Great Value Long Grain White Rice", "Salsa", "Squash", "Tomatillos", "Tomatoes"]

walmart_east_af_basket = ["Barley", "Curry Powder", "Dried Pinto Beans", "Dried Dates",
                          "Fava Beans", "Great Value Garbanzo Chick Peas", "Lentils",
                          "Great Value Basmati Rice", "Spaghetti Pasta", "Tomato Sauce", "Great Value Tuna"]

walmart_east_eu_basket = ["Barley", "Beef", "Beets", "Cabbage", "Lentils",
                          "Sliced Mushrooms", "Pork", "Potatoes", "Enriched Wheat Flour, 5lb"]

walmart_se_asia_basket = ["Bamboo Shoots", "Baby Corn", "Silk Coconut Milk", "Curry Paste", "Eggplant",
                          "Fish Sauce", "Vermicelli Rice Noodles", "Oyster Sauce", "Rice Flour",
                          "Brown Rice Noodles", "Great Value Jasmine Rice", "Sardines",
                          "Squash", "Tofu"]

walmart_west_af_basket = ["Great Value Black Eyed Peas, 16oz", "Brown Beans", "Cassava", "Chicken",
                          "Great Value Corn Flour", "Knorr Cube Bouillon Chicken 24ct", "Plantains",
                          "Potatoes", "Tomato Paste"]

# list of csv names
names = np.array(['Walmart_Universal_Food.csv', 'Walmart_Latinx_Food.csv', 'Walmart_East_Af_Food.csv',
 'Walmart_East_Eu_Food.csv', 'Walmart_SE_Asia_Food.csv', 'Walmart_West_Af_Food.csv'])
walmart_driver = food_helper.create_walmart_driver()
unique_stores = food_helper.search_zips(walmart_driver, zipcodes)
walmart_all_stores_baskets = food_helper.walmart_driver_func(walmart_driver, unique_stores, 20, walmart_foods, walmart_latinx_basket, walmart_east_af_basket, walmart_east_eu_basket, walmart_se_asia_basket, walmart_west_af_basket)
## NEED TO FIGURE OUT THE ERROR ABOVE.....<jupyter_output><empty_output><jupyter_text>Food Lion<jupyter_code># Food Lion Foods
fl_foods = ["Whole Milk", "12 Large Eggs", "Plain Bagels", "Cha Ching Bread",
            "Penne Pasta Rigate", "Gala Apples", "Food Lion Ground Turkey", "Food Lion Sliced Ham",
            "Butter", "Russet Potatoes Bag", "Navel Oranges Bag", "Tomatoes Vine",
            "Food Lion Lean Ground Beef 73%", "Food Lion Shredded Mexican Cheese", "Food Lion Yogurt", "Food Lion Cereal Honey Nut"]
#how to get cheaper whole milk

fl_latinx_basket = ["Hass Avocado", "Beets", "Food Lion Black Beans, 16oz", "Food Lion Shredded Mexican Cheese",
                    "Banderita Corn Tortilla", "Jalapenos", "Corn Flour",
                    "Green Bell Pepper", "Pinto Beans", "Food Lion White Rice",
                    "Food Lion Salsa Medium", "Squash", "Tomatillos", "Tomatoes Vine"]
# are canned beets ok?
# are we interested in canned or dry beans??

fl_east_af_basket = ["Quaker Barley", "Badia Curry Powder", "Pinto Beans",
                     "Food Lion Garbanzo Beans", "Lentils", "Mahatma Basmati Rice",
                     "Spaghetti Pasta", "Tomato Sauce", "Great Food Tuna"]

fl_east_eu_basket = ["Quaker Barley", "Food Lion Lean Ground Beef 73%", "Beets", "Buckwheat Grain",
                     "Cabbage", "Kasha", "Lamb Chuck", "Lentils", 
                     "Sliced Mushrooms", "Pork", "Russet Potatoes Bag", "Wheat Flour"]

fl_se_asia_basket = ["Bamboo Shoots", "Bean Thread", "Geisha Coconut Milk", "Curry Paste Patak", "Eggplant",
                     "Fish Sauce", "Vermicelli Rice Noodles", "Goya Rice Flour",
                     "Brown Rice Noodles", "Food Lion Jasmine Rice", "Squash", "Tapioca Flour"]

fl_west_af_basket = ["Black Eyed Peas, 16oz", "Pinto Beans", "La Reyna Yuca Cassava", "Food Lion Chicken",
                     "Corn Flour", "Food Lion Bouillon", "Plantains",
                     "Russet Potatoes Bag", "Food Lion Tomato Paste"]

baskets = [fl_foods, fl_latinx_basket, fl_east_af_basket, fl_east_eu_basket,
           fl_se_asia_basket, fl_west_af_basket]

names = np.array(['FL_Universal_Food.csv', 'FL_Latinx_Food.csv', 'FL_East_Af_Food.csv',
                  'FL_East_Eu_Food.csv', 'FL_SE_Asia_Food.csv', 'FL_West_Af_Food.csv'])

food_helper.food_lion(baskets, names, zipcodes)
# ARE PRICES ACTUAL PRICES OR INSTACART PRICES
### Walgreens Prices available in Store only...
### CVS missing lots of options
### Dollar General - prices in store only<jupyter_output><empty_output><jupyter_text>Amazon<jupyter_code># Amazon Fresh
af_foods = ["Gallon Whole Milk", "12 Eggland Large Eggs", "Fresh Brand Plain Bagels", "Bread",
            "Penne Pasta Rigate", "Fresh Brand 3 Gala Apples", "Ground Turkey", "Sliced Ham",
            "Amazon Brand Butter", "Potatoes", "Oranges", "Tomatoes Roma",
            "Ground Beef", "Mexican Cheese", "Amazon Brand Traditional Non-Fat Yogurt", "Fresh Brand Cereal 18.7"]
#how to get cheaper whole milk

af_latinx_basket = ["Hass Avocado", "Red Beets", "Black Beans", "Mexican Cheese",
                    "Corn Tortilla Amazon Brand", "Jalapenos", "Corn Flour",
                    "Green Bell Pepper", "Pinto Beans", "Happy Belly White Rice 5",
                    "Salsa Medium", "Squash", "Tomatillos", "Tomatoes Roma"]
# are canned beets ok?
# are we interested in canned or dry beans??

af_east_af_basket = ["Happy Belly Barley", "Amazon Brand Curry Powder", "Pinto Beans", "Dried Dates",
                     "Amazon Brand Garbanzo Beans", "Goya Lentils", "Basmati Rice",
                     "Amazon Brand Spaghetti Pasta 16", "Teff", "Amazon Brand Tomato Sauce 8", "Fresh Brand Tuna"]

af_east_eu_basket = ["Happy Belly Barley", "Ground Beef", "Red Beets",
                     "Green Cabbage", "Lamb", "Goya Lentils", 
                     "Fresh Brand White Mushrooms", "Pork", "Potatoes", "King Wheat Flour"]

af_se_asia_basket = ["Bamboo Shoots", "Baby Corn", "Bean Thread",
                     "Coconut Milk", "Curry Paste", "Amazon Brand Edamame", "Eggplant",
                     "Fish Sauce", "Rice Flour", "Taste of Thai Rice Noodles",
                     "Happy Belly Jasmine Rice", "Squash", "Tapioca Flour"]

af_west_af_basket = ["Black Eyed Peas", "Pinto Beans", "Chicken Drumsticks", "Corn Flour",
                     "Knorr Chicken Bouillon Cubes 24", "Potatoes", "Tomato Paste", "Red Yams"]
# what part of chicken...
# run Amazon driver
names = np.array(['AF_Universal_Food.csv', 'AF_Latinx_Food.csv', 'AF_East_Af_Food.csv',
                  'AF_East_Eu_Food.csv', 'AF_SE_Asia_Food.csv', 'AF_West_Af_Food.csv'])

af_baskets = [af_foods, af_latinx_basket, af_east_af_basket,
              af_east_eu_basket, af_se_asia_basket, af_west_af_basket]

food_helper.amazon(zipcodes, names, af_baskets)
# load af dataframes
af_df = pd.read_csv('AF_Universal_Food.csv', index_col = 0)
af_hispanic_df = pd.read_csv('AF_Latinx_Food.csv', index_col = 0)
af_east_af_df = pd.read_csv('AF_East_Af_Food.csv', index_col = 0)
af_east_eu_df = pd.read_csv('AF_East_Eu_Food.csv', index_col = 0)
af_se_asia_df = pd.read_csv('AF_SE_Asia_Food.csv', index_col = 0)
af_west_af_df = pd.read_csv('AF_West_AF_Food.csv', index_col = 0)
### BULK RETAILERS?<jupyter_output><empty_output><jupyter_text>BJ's<jupyter_code>bj_driver = webdriver.Chrome(ChromeDriverManager().install())
bj_driver.get("https://www.bjs.com/search")
bj_driver.maximize_window()
wait = WebDriverWait(bj_driver, driver_wait)

time.sleep(sleep_time)

change_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/span[2]')))
change_zip.click()
# change_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/span[2]')))
# change_zip.click()wa
#
store_loc = []
for zipcode in zipcodes:
    #
    enter_zip = wait.until(EC.element_to_be_clickable((By.ID, 'search')))
    enter_zip.clear()
    enter_zip.send_keys(str(zipcode), '\n')
    
    #
    element = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/div/app-header-club-modal-details-molecule/div[2]/div[1]/div/div[1]/div[1]/p[1]')))
    store_loc.append(element.text)
    
#
store_loc = np.array(store_loc)
_, idx = np.unique(store_loc, return_index = True)
bj_zipcodes = zipcodes[idx]

bj_foods = ["Whole Milk", "Large Eggs", "Wellsley Bagels", "Bread",
            "Penne Pasta Rigate", "Apples", "Turkey", "Ham",
            "Butter", "Russet Potatoes", "Navel Oranges", "Tomatoes",
            "Ground Beef", "Cheese", "Plain Yogurt", "Cinnamon Cereal"]
# only greek
# turkey is different for 2 stores... potatoes too

bj_latinx_basket = ["Beets", "Black Beans Bag", "Mexican Cheese", "Corn Flour",
                    "Bell Peppers", "Pinto Beans", "White Rice", "Medium Salsa", "Tomatoes"]
# selection is slightly different for corn flour

bj_east_af_basket = ["Curry Powder", "Pinto Beans", "Chickpeas", "Goya Lentils", "Basmati Rice",
                     "Spaghetti Pasta", "Tomato Sauce", "Tuna"]
# selection is slightly different for curr powder
# add if to handle this difference

bj_east_eu_basket = ["Ground Beef", "Beets", "Cabbage", "Lamb", "Goya Lentils", 
                     "Mushrooms", "Pork", "Russet Potatoes", "Whole Wheat Flour"]

bj_se_asia_basket = ["Goya Coconut Milk", "Edamame", "Jasmine Rice", "Sardines", "Tofu"]

bj_west_af_basket = ["Pinto Beans", "Chicken", "Corn Flour",
                     "Bouillon", "Plantains", "Russet Potatoes", "Tomato Paste"]
# what part of chicken...

bj_baskets = [bj_foods, bj_latinx_basket, bj_east_af_basket,
              bj_east_eu_basket, bj_se_asia_basket, bj_west_af_basket]

bj_driver = webdriver.Chrome(ChromeDriverManager().install())
bj_driver.get("https://www.bjs.com/search")
bj_driver.maximize_window()
wait = WebDriverWait(bj_driver, driver_wait)
time.sleep(sleep_time)

change_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/span[2]')))
change_zip.click()
time.sleep(sleep_time)

#
store_loc = []
all_prices = []
all_info = []
for zipcode in bj_zipcodes: 
    #
    enter_zip = wait.until(EC.element_to_be_clickable((By.ID, 'search')))
    enter_zip.clear()
    enter_zip.send_keys(str(zipcode), '\n')
    time.sleep(sleep_time)
    
    #
    element = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/div/app-header-club-modal-details-molecule/div[2]')))
    store_loc.append(", ".join(element.text.split("Miles")[0].split("\n")[1:-2]))
    
    #
    make_my_club = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'make-myclub')))
    make_my_club.click()
    time.sleep(sleep_time) # still didn't click..
    
    store_prices = []
    store_info = []
    for basket in bj_baskets[:2]:
        basket_prices = []
        basket_info = []

        for food in basket:
            #
            search = wait.until(EC.element_to_be_clickable((By.ID, 'searchTerm')))
            search.clear()
            search.send_keys(food, "\n")
            time.sleep(2)

            #
            if food_helper.check_exists_by_xpath(bj_driver, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[1]/app-search-header/div/div[1]/p[2]'):
                no_info = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[1]/app-search-header/div/div[1]/p[2]')))
                #
                if (no_info.text == "Here are some related results."):
                    price = np.nan
                    info = ""

                #
                else:
                    price = np.nan
                    info = "IDK WHAT HAPPENED"

            # too quick and does not line up with the new paths...
            elif food_helper.check_exists_by_xpath(bj_driver, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[2]/div[2]/div/app-products-container/div/div/div[1]/app-product-card/div/div'):
                element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[2]/div[2]/div/app-products-container/div/div/div[1]/app-product-card/div/div')))
                item = element.text.replace("Top Rated\n", "").replace("Best Seller\n", "")
                price = (float(item.split("\n$")[-1].split()[0]) + float(item.split("$")[-1].split()[-1])) / 2
                info = item.split("\n")[0]
                food_info.append(element.text)

            #
            else:
                item = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-pdp-preprocessor/div/app-pdp-layout-template/div/div/div/div[2]/div[1]/div[2]')))
                price = float(item.text.split("$")[1].split("\n")[0])
                info = item.text.split("\n")[0]

            #
            basket_prices.append(price)
            basket_info.append(info)
        
        #
        store_prices.append(basket_prices)
        store_info.append(basket_info)
    #
    change_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/span[2]')))
    change_zip.click()

    all_prices.append(store_prices)
    all_info.append(store_info)
    
#
store_loc = np.array(store_loc)
#
bj_driver = webdriver.Chrome(ChromeDriverManager().install())
bj_driver.get("https://www.bjs.com/search")
bj_driver.maximize_window()
wait = WebDriverWait(bj_driver, driver_wait)
time.sleep(2 * sleep_time)

#
store_loc = []
all_prices = []
all_info = []
for zipcode in bj_zipcodes: 
    #
    change_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/span[2]')))
    change_zip.click()
    time.sleep(2 * sleep_time)
    
    #
    enter_zip = wait.until(EC.element_to_be_clickable((By.ID, 'search')))
    enter_zip.clear()
    enter_zip.send_keys(str(zipcode), '\n')
    time.sleep(sleep_time)
    
    #
    element = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="header-margin"]/ul/li[4]/app-header-find-aclub-link-molecule/div/div[2]/div/app-header-club-modal-details-molecule/div[2]')))
    store_loc.append(", ".join(element.text.split("Miles")[0].split("\n")[1:-2]))
    
    #
    make_my_club = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'make-myclub')))
    make_my_club.click()
    time.sleep(sleep_time) # still didn't click..
    
    #
    store_prices = []
    store_info = []
    for basket in bj_baskets[:2]:
        
        #
        basket_prices = []
        basket_info = []
        for food in basket:
            #
            search = wait.until(EC.element_to_be_clickable((By.ID, 'searchTerm')))
            search.clear()
            search.send_keys(food, "\n")
            time.sleep(2)

            #
            if food_helper.check_exists_by_xpath(bj_driver, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[1]/app-search-header/div/div[1]/p[2]'):
                no_info = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[1]/app-search-header/div/div[1]/p[2]')))
                #
                if (no_info.text == "Here are some related results."):
                    price = np.nan
                    info = ""

                #
                else:
                    price = np.nan
                    info = "IDK WHAT HAPPENED"

            # too quick and does not line up with the new paths...
            elif food_helper.check_exists_by_xpath(bj_driver, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[2]/div[2]/div/app-products-container/div/div/div[1]/app-product-card/div/div'):
                element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-search-main/app-search-result-page-gb/div[2]/div[2]/div/app-products-container/div/div/div[1]/app-product-card/div/div')))
                item = element.text.replace("Top Rated\n", "").replace("Best Seller\n", "")
                price = (float(item.split("\n$")[-1].split()[0]) + float(item.split("$")[-1].split()[-1])) / 2
                info = item.split("\n")[0]
                food_info.append(element.text)

            #
            else:
                #
                item = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/app-root/div/div[2]/div/app-pdp-preprocessor/div/app-pdp-layout-template/div/div/div/div[2]/div[1]/div[2]')))
                info = item.text.split("\n")[0]
                
                #
                if "SOLD OUT" in item.text:
                    price = np.nan

                #
                else:
                    price = float(item.text.split("$")[1].split("\n")[0])

            #
            basket_prices.append(price)
            basket_info.append(info)
        
        #
        store_prices.append(basket_prices)
        store_info.append(basket_info)

    #
    all_prices.append(store_prices)
    all_info.append(store_info)
    
#
store_loc = np.array(store_loc)<jupyter_output>====== WebDriver manager ======
Current google-chrome version is 92.0.4515
Get LATEST driver version for 92.0.4515
Driver [/Users/zacharyalerte/.wdm/drivers/chromedriver/mac64/92.0.4515.107/chromedriver] found in cache<jupyter_text>Sam's Club<jupyter_code># Sam's Club
sam_foods = ["Milk", "Grade A Large White Eggs", "Member's Mark Plain Bagels", "Bread",
            "Penne Pasta Rigate", "Gala Apples", "Ground Turkey", "Sliced Ham",
            "Butter", "Russet Potatoes", "Large Seedless Oranges", "Tomatoes Roma",
            "Ground Beef 80%", "Mexican Cheese", "Nonfat Plain Yogurt", "Cereal Honey"]
#only greek yogurt

sam_latinx_basket = ["Avocado", "Ground Beef 80%", "Member's Mark Black Beans", "Mexican Cheese",
                    "Corn Tortilla", "Jalapenos", "Masa",
                    "Green Bell Peppers", "Member's Mark Pinto Beans", "White Rice",
                    "Salsa Medium", "Squash", "Tomatillos", "Tomatoes Roma"]
# what size for rice?? 25 or 50lbs... // same with salsa medium

sam_east_af_basket = ["Curry Powder", "Member's Mark Pinto Beans", "Chickpeas", "Basmati Rice",
                     "Spaghetti Pasta", "Tomato Sauce", "Member's Mark Tuna"]

sam_east_eu_basket = ["Ground Beef 80%", "Lamb", "Goya Lentils", 
                     "Mushrooms", "Pork", "Russet Potatoes", "Wheat Flour"]

sam_se_asia_basket = ["Coconut Milk", "Rice Flour", "Taste of Thai Rice Noodles",
                      "Member's Mark Jasmine Rice", "Sardines", "Squash"]

sam_west_af_basket = ["Member's Mark Pinto Beans", "Chicken Breast", "Masa",
                     "Bouillon", "Russet Potatoes", "Tomato Paste"]
# what part of chicken...

sam_baskets = [sam_foods, sam_latinx_basket, sam_east_af_basket,
              sam_east_eu_basket, sam_se_asia_basket, sam_west_af_basket]

sam_driver = webdriver.Chrome(ChromeDriverManager().install())
sam_driver.get("https://www.samsclub.com/s/")
sam_driver.maximize_window()

# also has reCAPTCHA
names = np.array(['Sam_Universal_Food.csv', 'Sam_Latinx_Food.csv', 'Sam_East_Af_Food.csv',
                  'Sam_East_Eu_Food.csv', 'Sam_SE_Asia_Food.csv', 'Sam_West_Af_Food.csv'])

sams_club(sam_driver, zipcodes, names, 20, 2, sam_foods, sam_latinx_basket, sam_east_af_basket,
              sam_east_eu_basket, sam_se_asia_basket, sam_west_af_basket)
## NEED TO FIGURE OUT LOCATION.. GETTING GENERAL NAME ISN'T TOUGH BUT WHAT ABOUT OTHER STORES...

# location = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div[2]/div[1]/div/div/div[2]/div/nav[2]/div/div[2]/div/div/button')))
# location.click()
universal_df_copy = universal_df.copy()
universal_df_copy["name"] = universal_df_copy["Store"]
universal_df_copy = universal_df_copy.drop(["Item", "Price", "Other_Info", "Store"], axis = 1)
universal_df_copy.head()
address = np.array([re.sub("^\\D*", "", universal_df_copy.Location.iloc[i]).split(",")[0] for i in range(len(universal_df_copy))])
most_info = np.array([re.sub("^\\D*", "", universal_df_copy.Location.iloc[i]) for i in range(len(universal_df_copy))])

zipcode = [int(most_info[i].split(" ")[-1][:5]) for i in range(len(most_info))] # good
state = [most_info[i].split(" ")[-2].strip(",") for i in range(len(most_info))] # includnig washington in DC?
city = [most_info[i].split(",")[-2].strip(" ") for i in range(len(most_info))] # good

universal_df_copy['Address'] = address
universal_df_copy['City'] = city
universal_df_copy['State'] = state
universal_df_copy['zipcode'] = zipcode
universal_df_copy = universal_df_copy.drop(["Location"], axis = 1)
universal_df_copy.head()
universal_df_copy.to_csv("Store_Locations_df.csv")
# !pip install geopandas
# !pip install geopy
universal_df_copy = pd.read_csv("Store_Locations_df.csv", index_col = 0)
universal_df_copy_short = universal_df_copy.drop_duplicates().reset_index(drop=True)
universal_df_copy_short["City"] = [universal_df_copy_short["City"][i].replace(" VA", "").replace(" DC", "") for i in range(len(universal_df_copy_short))]
short_address = universal_df_copy_short["Address"]
short_city = universal_df_copy_short["City"]
short_state = universal_df_copy_short["State"]
short_zipcode = universal_df_copy_short["zipcode"]
# errors = 0
# from geopy.geocoders import Nominatim
# locator = Nominatim(user_agent="myGeocoder")

# idxs = []
# lat = []
# long = []
# for i in range(len(short_address)):
#     #
#     try:
#         location = locator.geocode("{}, {}, {} {}".format(short_address[i], short_city[i], short_state[i], short_zipcode[i]))
#         print("{}, {}, {} {}".format(short_address[i], short_city[i], short_state[i], short_zipcode[i]))
#         print("Latitude = {}, Longitude = {}".format(location.latitude, location.longitude))
#         idxs.append(i)
#         lat.append(location.latitude)
#         long.append(location.longitude)
    
#     #
#     except:
#         #
#         print("Error at {}".format(i))
#         errors += 1
#         continue
        
# #
# print("\nThere are {} errors.".format(errors))
universal_df_copy_short_ll = universal_df_copy_short.iloc[np.array(idxs)].copy()
universal_df_copy_short_ll["Latitude"] = lat
universal_df_copy_short_ll["Longitude"] = long
universal_df_copy_short_ll.head()

universal_df_copy_short_ll.to_csv("Store_Locations_df_w_ll.csv")

#trying to get latitude and longitude data for stores
big_location_df = pd.read_csv("export_w_target.csv", index_col = 0)
basic_stores = ["whole foods market", "harris teeter", "aldi", "giant", "safeway", "target"]
basic_stores_location_df = big_location_df[big_location_df['brand'].str.lower().isin(basic_stores)]
narrow_basic_stores_location_df = basic_stores_location_df[['brand', 'lat', 'long']].reset_index(drop = True)
narrow_basic_stores_location_df.to_csv("Narrow_basic_stores_location_df.csv")
# Aldi does not have zipcodes or towns...
# Target info is a bit werid but can be cleaned

# HMart
giant_se_asia_foods = ["Coconut Milk", "Fish Sauce", "Vermicelli", "Oyster Sauce",
                       "Rice Flour", "Rice Noodles", "Jasmine Rice"]

# start hmart driver
hmart_driver = webdriver.Chrome(ChromeDriverManager().install())
hmart_driver.get("https://www.hmart.com/")
hmart_driver.maximize_window()
wait = WebDriverWait(hmart_driver, 20)

#
all_info = []
random_zips = zipcodes[::5]
for random_zip in random_zips:
    #
    click_zipcode = wait.until(EC.element_to_be_clickable((By.XPATH, "//header/div[2]/div/div/div[2]/div/div[1]")))
    click_zipcode.click()

    #
    send_zip = wait.until(EC.element_to_be_clickable((By.ID, "zipcode_hfresh")))
    send_zip.clear()
    send_zip.send_keys(str(random_zip), "\n")
    
    #
    info = []
    for food in giant_se_asia_foods:
        #
        search = wait.until(EC.element_to_be_clickable((By.ID, "search")))
        search.clear()
        search.send_keys(food, "\n")

        #
        element = wait.until(EC.element_to_be_clickable((By.XPATH, "//ol/li[1]/div/div")))
        info.append(element.text.replace("\nAdd to Cart", "").replace("\nSALE", ""))

    #
    all_info.append(info)
    
# close hmart driver
hmart_driver.quit()

#
all_info = np.array(all_info)

hmart_prices = np.array([[float(all_info[j][i].split("\n")[-1].split(" ")[0].replace("$", "")) for i in range(len(all_info[j]))] for j in range(len(all_info))])<jupyter_output><empty_output><jupyter_text>Kroger's<jupyter_code>krogers_driver = webdriver.Chrome(ChromeDriverManager().install())
krogers_driver.get("https://www.kroger.com/")
krogers_driver.maximize_window()
# throwing strange errors...
wait = WebDriverWait(krogers_driver, 20)
search = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="SearchBar-input-open"]')))
search.clear()
search.send_keys("Whole Milk", "\n")<jupyter_output><empty_output><jupyter_text>Wegmans<jupyter_code># Wegmans
wegmans_basket = ["Whole Milk", "Large Eggs", "Plain Bagels", "Bread",
                  "Wegmans Penne", "Gala Apples", "Ground Turkey", "Ham",
                  "Wegmans Butter", "Wegmans Russet Potatoes", "Oranges", "Tomatoes",
                  "Ground Beef", "Mexican Cheese", "Wegmans Plain Lowfat Yogurt", "Cereal Honey"]

wegmans_latinx_basket = ["Avocado", "Ground Beef", "Black Beans", "Mexican Cheese",
                         "Tortillas Corn", "Jalapenos", "Instant Corn Masa",
                         "Green Bell Peppers", "Pinto Beans", "Long Grain White Rice",
                         "Wegmans Medium Salsa", "Green Squash", "Tomatillos", "Tomatoes Roma"]
# what size for rice?? 25 or 50lbs... // same with salsa medium

wegmans_east_af_basket = ["Laxmi Curry Powder", "Pinto Beans", "Chickpeas", "Dried Dates",
                          "Fava Beans", "Goat Cubes", "Goya Lentils", "Wegmans Basmati Rice",
                          "Spaghetti Pasta", "Tomato Sauce", "Tuna"]
# maybe sort for cheap curry powder

wegmans_east_eu_basket = ["Barley", "Ground Beef", "Beets", "Lamb", "Goya Lentils", "Cabbage", "Kasha",
                          "Mushrooms", "Pork", "Wegmans Russet Potatoes", "Wheat Flour", "Veal"]

wegmans_se_asia_basket = ["Bamboo Shoots", "Baby Corn", "Bean Threads", "Silk Coconut Milk",
                          "Curry Paste", "Edamame", "Eggplant", "Fish Sauce",
                          "Vermicelli", "Oyster Sauce", "Rice Flour", "Thai Rice Noodles",
                          "Wegmans Jasmine Rice", "Sardines", "Squash", "Tapioca Flour", "Tofu"]

wegmans_west_af_basket = ["Black Eyed Peas", "Cassava", "Chicken Breast",  "Instant Corn Masa",
                          "Goat Cubes", "Bouillon Cubes", "Palm Oil", "Dole Plantains",
                          "Wegmans Russet Potatoes", "Tomato Paste", "Yams"]
# what part of chicken...

wegmans_baskets = [wegmans_foods, wegmans_latinx_basket, wegmans_east_af_basket,
                   wegmans_east_eu_basket, wegmans_se_asia_basket, wegmans_west_af_basket]

zip_df = pd.read_csv("csvData.csv")
all_va_zips = zip_df.zip
wegman_cities = ['Alexandria', 'Chantilly', 'Charlottesville', 'Dulles', 'Fairfax',
                 'Fredericksburg', 'Lake Manassas', 'Leesburg', 'Midlothian', 'Potomac',
                 'Short Pump', 'Tysons', 'Virginia Beach']
### NEED TO ADD IN OTHER CULTURAL BASKETS!! - GOOD THING IS THAT WE HAVE INFO ON THE ENTIRE STATE!
wegmans_driver = webdriver.Chrome(ChromeDriverManager().install())
wegmans_driver.get("https://shop.wegmans.com/shop/categories")
wegmans_driver.maximize_window()
wait = WebDriverWait(wegmans_driver, 20)

in_store = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div/div/div[2]/div/div/div[2]/span/shopping-context-item[3]/button')))
in_store.click()

all_info = []
for city in wegman_cities:
    search = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div[2]/div/header/div[2]/div[1]/div/div[2]/unata-shopping-selector-nav/div/div/react-context-selector/div/div[3]/button')))
    search.click()
    
    enter_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div/div/div[2]/div/div/div/div/div[1]/div[2]/form/div/input')))
    enter_zip.clear()
    enter_zip.send_keys(city, "\n")
    
    select = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div/div/div[2]/div/div/div/div/div[2]/div[2]/div/div/div/div/div[2]/button')))
    select.click()
    time.sleep(sleep_time)
    
    info = []
    for food in wegmans_basket:
        search = wait.until(EC.element_to_be_clickable((By.ID, 'search-nav-input')))
        search.clear()
        search.send_keys(food, "\n")
        time.sleep(sleep_time)

        element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'cell-content-wrapper')))
        info.append(element.text)
    
    all_info.append(info)

item = np.array([[all_info[j][i].split("\n")[0] for i in range(len(all_info[j]))] for j in range(len(all_info))])
price = np.array([[float(all_info[j][i].split("$")[1].split(" ")[0]) for i in range(len(all_info[j]))] for j in range(len(all_info))])
    
wegmans_driver.quit()<jupyter_output>====== WebDriver manager ======
Current google-chrome version is 93.0.4577
Get LATEST driver version for 93.0.4577
Driver [/Users/zacharyalerte/.wdm/drivers/chromedriver/mac64/93.0.4577.15/chromedriver] found in cache<jupyter_text>Example:<jupyter_code># check if xpath exists, if not return false
def check_exists_by_xpath(driver, xpath):
    '''
    Description:
        Check existence of xpath on page
    
    Inputs:
        webdriver: your webdriver
        xpath: whatever element we are looking for
        
    Outputs:
        returns True if xpath exists, False if not
    '''
    # try to find element
    try:
        driver.find_element_by_xpath(xpath)
    
    # throw exception and return false if unable to find
    except NoSuchElementException:
        return False
    return True

### TARGET
def target_locations(zipcodes, sleep_time = 2, driver_wait = 20):
    '''
    Description:
        Identify unique Targets in the Arlington Area
    
    Inputs:
        zipcodes:
        sleep_time: integer, system sleep time between certain processes, default = 2
        driver_wait: integer, wait time for driver - default = 20
        
    Outputs:
        target_zipcodes: array of zipcodes shortened to identify unique Targets
        unique_target_locs: array of Target locations
    '''
    # Create Target driver and visit website
    target_driver = webdriver.Chrome(ChromeDriverManager().install())
    target_driver.maximize_window()
    target_driver.get("https://www.target.com/c/grocery/-/N-5xt1a")

    # select store option to begin passing in zipcodes
    wait = WebDriverWait(target_driver, driver_wait)
    store = wait.until(EC.element_to_be_clickable((By.ID, 'storeId-utilityNavBtn')))
    store.click()

    # loop over zipcodes to collect information on store location
    target_info = []
    for zipcode in zipcodes:
        # click on option to edit zipcode
        edit = wait.until(EC.element_to_be_clickable((By.ID, 'zipOrCityState')))
        edit.click()

        # delete zipcode then send new zipcode (not sure why enter_zip.clear() did not work)
        enter_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[9]/div/div/div/div/div[1]/div/div[3]/div[1]/div/input')))
        enter_zip.send_keys('\<KEY>')
        enter_zip.send_keys(str(zipcode), "\n")

        # extract store location information
        element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[9]/div/div/div/div/div[3]')))
        target_info.append(element.text)

    # close driver
    target_driver.quit()

    # get unique locations and corresponding zipcodes
    unique_target_locs, target_idx = np.unique(target_info, return_index = True)
    target_zipcodes = zipcodes[target_idx]
    
    # clean location information
    for i, loc in enumerate(unique_target_locs):
        unique_target_locs[i] = loc.split("\n")[1]
    
    #
    return target_zipcodes, unique_target_locs

def target_driver(target_zipcodes, target_foods, sleep_time = 2, driver_wait = 20, standard = True):
    '''
    Description:
        Extract item information for specific stores for searched food items
    
    Inputs:
        target_zipcodes: array of zipcodes to enter to identify closest stores
        target_foods: array of food items to search for on website
        sleep_time: integer, system sleep time between certain processes, default = 2
        driver_wait: integer, wait time for driver - default = 20
        standard: Boolean, True if using "Universal" basket of goods
        
    Outputs:
        target_items_by_zip: 2D array of food items by store (contains other information)
    '''
    # start driver and visit Target side
    target_driver = webdriver.Chrome(ChromeDriverManager().install())
    target_driver.maximize_window()
    target_driver.get("https://www.target.com/c/grocery/-/N-5xt1a")

    # loop over all unique Targets
    target_items_by_zip = []
    for k, zipcode in enumerate(target_zipcodes):
        # click on option to select a store
        wait = WebDriverWait(target_driver, driver_wait)
        store = wait.until(EC.element_to_be_clickable((By.ID, 'storeId-utilityNavBtn')))
        store.click()

        # edit and then add zipcode
        edit = wait.until(EC.element_to_be_clickable((By.ID, 'zipOrCityState')))
        edit.click()

        # erase old zipcode and send new one
        enter_zip = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[9]/div/div/div/div/div[1]/div/div[3]/div[1]/div/input')))
        enter_zip.send_keys('\<KEY>b')
        enter_zip.send_keys(str(zipcode), "\n")

        # select the top store
        go_top_store = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[9]/div/div/div/div/div[3]/div[2]/div[1]/button')))
        go_top_store.click()

        # loop over staples at store
        targ_info = []
        for food in target_foods:
            # selects next closest beef option for this particular store
            if (food == "Ground Beef 1lb 80") and (k == 4):
                food = "Ground Beef 1lb 73"

            # search for food item
            target_driver.get("https://www.target.com/s?searchTerm={}".format(food))
            
            # checks if search item is tomato sauce to sort by price
            if (food == "Tomato Sauce 8oz"):
                sort = wait.until(EC.element_to_be_clickable((By.XPATH, "/html/body/div[1]/div/div[4]/div[4]/div[2]/div/div[2]/div[3]/div[1]/div[2]/div[2]/div/div[2]/button")))
                sort.click()
                by_price = wait.until(EC.element_to_be_clickable((By.XPATH, "//li[3]/a/div/div")))
                by_price.click()
                time.sleep(sleep_time)
            
            # checks if search item is NOT plantains (these are in different section for some reason...)
            if (food != "Platano") and (check_exists_by_xpath(target_driver, '//*[@id="mainContainer"]/div[4]/div[2]/div/div[2]/div[3]/div/ul/li[1]/div/div[2]/div')):
            # extract slightly cleaned pricing info
#                 info = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div[4]/div[4]/div[2]/div/div[2]/div[3]/div/ul/li[1]/div/div[2]/div/div/div')))
                info = wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="mainContainer"]/div[4]/div[2]/div/div[2]/div[3]/div/ul/li[1]/div/div[2]/div')))
#/html/body/div[1]/div/div[4]/div[4]/div[2]/div/div[2]/div[3]/div/ul/li[1]/div/div[2]/div/div/div
            
            # if food item is plantain - grab info from bottom of webpage
            else:
                info = wait.until(EC.element_to_be_clickable((By.XPATH, "/html/body/div[1]/div/div[4]/div[4]/div[2]/div/div[1]/div[3]/div[2]/div/ul/li[1]/div/div[2]/div/div/div")))
                
            #
            cleaned_info = "".join(re.split('(.\d\d)', info.text)[:-1])
            targ_info.append(cleaned_info)

        # append list of items to list and close driver
        target_items_by_zip.append(targ_info)
    target_driver.quit()
    
    # convert from list to array
    target_items_by_zip = np.array(target_items_by_zip)
    return target_items_by_zip

def target_price_item_other(target_items_by_zip):
    '''
    Description:
        Extracts price, item info, and other information from the 2D array of information passed in
    
    Inputs:
        target_items_by_zip: 2D array of food items by store (contains other information)
    
    Outputs:
        prices_target: 2D array of prices for each item at each store
        cleaned_items_target: 2D array of products at each store
        other_info_target: 2D array of additional product information
    '''
    # clean strings and separate price, items, and other information
    cleaned_target = np.array([np.array([target_items_by_zip[j][i] for i in range(len(target_items_by_zip[j]))]) for j in range(len(target_items_by_zip))])
    cleaned_items_target = np.array([np.array([cleaned_target[j][i].split("\n")[0] for i in range(len(cleaned_target[j]))]) for j in range(len(cleaned_target))])
    
    # heinous
#     prices_target = np.array([np.array([float(cleaned_target[j][i].split("\n")[-3].split(" ")[0].replace("$", "")) if cleaned_target[j][i].split("\n")[-1][:4] == "Free" else float(cleaned_target[j][i].split("\n")[-2].split(" ")[0].replace("$", "")) if cleaned_target[j][i].split("\n")[-1][:3] == "Buy" else float(cleaned_target[j][i].split("\n")[-2].split(" ")[0].replace("$", "")) if cleaned_target[j][i].split("\n")[-1][:3] == "Get" else float(cleaned_target[j][i].split("\n")[-1].split(" ")[0].replace("$", "")) for i in range(len(cleaned_target[j]))]) for j in range(len(cleaned_target))])
    
    prices_target = np.array([np.array([float(re.sub("[^0-9\.]", "", cleaned_target[j][i].split("\n$")[-1].split(" ")[0])) for i in range(len(cleaned_target[j]))]) for j in range(len(cleaned_target))])
    other_info_target = np.array([np.array(["" for i in range(len(cleaned_target[j]))]) for j in range(len(cleaned_target))])
    return prices_target, cleaned_items_target, other_info_target

def target(foods, zipcodes, standard = True, has_zipcodes = False, locs = []):
    '''
    Description:
        Runs combination of functions to go from a list of zipcodes and food items to final dataframe for Target
    
    Inputs:
        foods: array of food items
        zipcodes: list of Arlington zipcodes
        standard: Boolean, True if using "Universal" basket of goods
    
    Outputs:
        df: dataframe with price, info, etc.
    '''
    # if shortened zipcodes list passed in
    if has_zipcodes:
        shortened_zipcodes = zipcodes
        shortened_locations = locs
    
    # get shortened list of zipcodes if not provided
    else:
        shortened_zipcodes, shortened_locations = target_locations(zipcodes)
    
    items_by_zip = target_driver(shortened_zipcodes, target_foods = foods, standard = standard)
    prices_target, cleaned_items_target, other_info_target = target_price_item_other(items_by_zip)
    df = make_df(prices_target, cleaned_items_target, other_info_target, shortened_locations, "Target")
    
    # if we already have zipcodes and locations, return dataframe (else return all 3)
    if has_zipcodes:
        return df
    
    return df, shortened_zipcodes, shortened_locations

target_zipcodes, unique_target_locs = target_locations(zipcodes)
target_items_by_zip = target_driver(target_zipcodes, target_foods)
prices_target, cleaned_items_target, other_info_target = target_price_item_other(target_items_by_zip)
# hispanic scraping

target_hispanic_foods_big = ["Avocado",
                             "Beets",
                             "Good and Gather Black Beans",
                             "Mexican Cheese",
                             "Corn Tortillas",
                             "Crackers",
                             "Green Chiles",
                             "Jalapeno Peppers",
                             "Garbanzo Beans",
                             "Masa",
                             "Fruit Cocktail Market",
                             "Nopalitos",
                             "Bell Peppers",
                             "Pinto Beans",
                             "White Rice 5",
                             "Medium Salsa 24",
                             "Squash",
                             "Tomatillos",
                             "Good and Gather Salsa Verde",
                             "Roma Tomatoes"]

target_hispanic_items_by_zip = target_driver(target_zipcodes, target_hispanic_foods_big, standard = False)
# probably need to select in-store pickup??? really not clear here<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup