<jupyter_start><jupyter_text>Glassdoor website scraping  Import<jupyter_code>import pandas as pd
import numpy as np

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
import time

pd.set_option('display.max_columns', None)<jupyter_output><empty_output><jupyter_text>Set up chrome as PATH and set up drivers<jupyter_code>PATH = "C:\Program Files\chromedriver.exe"
driver = webdriver.Chrome(PATH)
driver_comp = webdriver.Chrome(PATH)
driver_benefits = webdriver.Chrome(PATH)<jupyter_output><empty_output><jupyter_text>Scraping Glassdoor.com<jupyter_code>#def get_data(page_start,page_end):
companies_all = []
for page in range(1,999):
    driver.get("https://www.glassdoor.com/Explore/browse-companies.htm?overall_rating_low=0&page="+str(page)+"&isHiringSurge=0")
    main = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.ID, 'ReactCompanyExplorePageContainer')))
    time.sleep(2)
    # company list per page (10 companies each page)
    comp = driver.find_element_by_xpath("//*[@id='ReactCompanyExplorePageContainer']/div/div/div/div/div[2]")
    print('-----------------------------------------------------------------begin of ',page)
    for i in range(1,11):
        
        try:
            name = comp.find_element_by_xpath("//section["+str(i)+"]//h2[@data-test = 'employer-short-name']")
            name_text = name.text
        except NoSuchElementException:
            name_text = np.nan

        try:
            indus = comp.find_element_by_xpath("//section["+str(i)+"]//span[@data-test='employer-industry']")
            indus_text = indus.text
        except NoSuchElementException:
            indus_text = np.nan

        try:
            rating = comp.find_element_by_xpath("//section["+str(i)+"]//span[@data-test='rating']")
            rating_text = rating.text
        except NoSuchElementException:
            rating_text = np.nan

        try:
            office = comp.find_element_by_xpath("//section["+str(i)+"]//span[@data-test='employer-location']/a")
            office_text = office.text
        except NoSuchElementException:
            office_text = np.nan

        try:
            size = comp.find_element_by_xpath("//section["+str(i)+"]//span[@data-test='employer-size']")
            size_text = size.text
        except NoSuchElementException:
            size_text = np.nan

        try:
            reviews_count = comp.find_element_by_xpath("//section["+str(i)+"]//div[@data-test='cell-Reviews-count']")
            reviews_count_text = reviews_count.text
        except NoSuchElementException:
            reviews_counts_text = np.nan

        try:
            salaries_count = comp.find_element_by_xpath("//section["+str(i)+"]//div[@data-test='cell-Salaries-count']")
            salaries_count_text = salaries_count.text
        except NoSuchElementException:
            salaries_count_text = np.nan

        try:
            jobs_count = comp.find_element_by_xpath("//section["+str(i)+"]//div[@data-test='cell-Jobs-count']")
            jobs_count_text = jobs_count.text
        except NoSuchElementException:
            jobs_count_text = np.nan

        try:
            desc = comp.find_element_by_xpath("//section["+str(i)+"]//p[@class='employerCard__EmployerCardStyles__clamp common__commonStyles__subtleText']")
            desc_text = desc.text
        except NoSuchElementException:
            desc_text = np.nan

    # Company overview link
        try:
            overview_link = comp.find_element_by_xpath("//section["+str(i)+"]/div/div[6]/div/a").get_attribute('href')
        except NoSuchElementException:
            overview_link = np.nan
            continue

    #company reviews, salaries, jobs links
        try:
            reviews_link = comp.find_element_by_xpath("//section["+str(i)+"]//a[1]").get_attribute('href')
        except NoSuchElementException:
            review_link = np.nan

        try:
            salaries_link = comp.find_element_by_xpath("//section["+str(i)+"]//a[2]").get_attribute('href')
        except NoSuchElementException:
            salaries_link = np.nan

        try:
            jobs_link = comp.find_element_by_xpath("//section["+str(i)+"]//a[3]").get_attribute('href')
        except NoSuchElementException:
            jobs_link = np.nan

    # Go to overview link -------------------------------------------------------------------   
        driver_comp.get(overview_link)
        main = WebDriverWait(driver_comp,10).until(
            EC.presence_of_element_located((By.ID, 'EIOverviewContainer'))
            )
            
        time.sleep(2)
    #----------------------------------------------------------------------------------------    

    # Get data from overview
        try:
            interviews_count_text = driver_comp.find_element_by_xpath("//*[@id='EIProductHeaders']/div/a[4]/span[1]").text
        except NoSuchElementException:
            interviews_count_text = np.nan

        try:
            benefits_count_text = driver_comp.find_element_by_xpath("//*[@id='EIProductHeaders']/div/a[5]/span[1]").text
        except NoSuchElementException:
            benefits_count_text = np.nan


        ov_container = driver_comp.find_element_by_xpath("//*[@id='EIOverviewContainer']")

        try:
            website = ov_container.find_element_by_xpath("//a[@data-test = 'employer-website']").get_attribute('href')
        except NoSuchElementException:
            website = np.nan

        try:
            comp_type_text = ov_container.find_element_by_xpath("//div[@data-test = 'employer-type']").text
        except NoSuchElementException:
            comp_type_text = np.nan

        try:
            revenue_text = ov_container.find_element_by_xpath("//div[@data-test = 'employer-revenue']").text
        except NoSuchElementException:
            revenue_text = np.nan

        try:
            hq_text = ov_container.find_element_by_xpath("//div[@data-test = 'employer-headquarters']").text
        except NoSuchElementException:
            hq_text = np.nan

        try:
            founded_text = ov_container.find_element_by_xpath("//div[@data-test = 'employer-founded']").text
        except NoSuchElementException:
            founded_text = np.nan

        try:
            interview_pos_text = ov_container.find_element_by_xpath("//div[@data-test = 'interviewStatsChart']//div[@class='d-flex']/div[2]/div[1]/span[2]").text
        except NoSuchElementException:
            interview_pos_text = np.nan

        try:
            interview_neg_text = ov_container.find_element_by_xpath("//div[@data-test = 'interviewStatsChart']//div[@class='d-flex']/div[2]/div[2]/span[2]").text
        except NoSuchElementException:
            interview_neg_text = np.nan

        try:
            interview_neut_text = ov_container.find_element_by_xpath("//div[@data-test = 'interviewStatsChart']//div[@class='d-flex']/div[2]/div[3]/span[2]").text
        except NoSuchElementException:
            interview_neut_text = np.nan

        try:
            interview_difficulty_text = ov_container.find_element_by_xpath("//div[@data-test = 'interviewDifficultyContainer']//div[@class='css-155sv15 e1webdg50']").text
        except NoSuchElementException:
            interview_difficulty_text = np.nan

        # Go to benefits page  ----------------------------------------------------------------  
        benefits_page = driver_comp.find_element_by_xpath("//*[@id='EIProductHeaders']/div/a[5]").get_attribute('href')
        driver_benefits.get(benefits_page)
        main = WebDriverWait(driver_benefits,10).until(
            EC.presence_of_element_located((By.ID, 'NodeReplace'))
            )
        time.sleep(2)
        #--------------------------------------------------------------------------------------
        
        # Benefits score
        try:
            benefits_score = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[1]/div[2]/div/div[1]/strong").text
        except NoSuchElementException:
            benefits_score = np.nan


        # Insurance, Health & Wellness (13 benefits)
        insurance_health_wellness = 0
        insurance_health_wellness_ls = ['total_insurance_health_wellness_benefits','health_insurance','dental_insurance',
                                       'FSA','vision insurance','HSA','life_insurance','supplemental_life_insurance',
                                       'disability_insurance','occupational_accident_insurance','heakth_care_on_site',
                                       'mental_health_care','retiree_health_and_medical',
                                        'accidental_death_and_dismemberment_insurance']
        insurance_health_wellness_val_ls = []
        for ind in range(1,14):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[1]/ul/li["+str(ind)+"]/a").get_attribute('title')
                insurance_health_wellness +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0 
            insurance_health_wellness_val_ls.append(does_have)
        insurance_health_wellness_val_ls.insert(0,insurance_health_wellness) 
        insurance_health_wellness_dict ={insurance_health_wellness_ls[ind] : insurance_health_wellness_val_ls[ind] for ind in range(0,14)}    

        # financial & Retirement (9 benefits)
        financial_retirement = 0
        financial_retirement_ls = ['total_financial_retirement_benefits','pension_plan','plan_401K',
                                  'retirement_plan','employee_stock_purcase_plan', 'performance_bonus',
                                  'stock_options','equity_incentive_plan','supplemental_workers_compensation',
                                  'charity_gift_matching']
        financial_retirement_val_ls = []
        for ind in range(1,10):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[2]/ul/li["+str(ind)+"]/a").get_attribute('title')
                financial_retirement +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0
            financial_retirement_val_ls.append(does_have)
        financial_retirement_val_ls.insert(0,financial_retirement)   
        financial_retirement_dict ={financial_retirement_ls[ind] : financial_retirement_val_ls[ind] for ind in range(0,10)}

        # Family & Parenting (10 benefits)
        family_parenting = 0
        family_parenting_ls = ['total_family_parenting_benefits','maternity_paternity_leave','work_from_home', 'fertility_assistance', 'dependent_care',
                              'adoption_assistance', 'childcare','flexible_hour','military_leave',
                               'family_medical_leave', 'unpaid_extended_leave']
        family_parenting_val_ls = []

        for ind in range(1,11):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[3]/ul/li["+str(ind)+"]/a").get_attribute('title')
                family_parenting +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0
            family_parenting_val_ls.append(does_have)
        family_parenting_val_ls.insert(0,family_parenting)   
        family_parenting_dict ={family_parenting_ls[ind] : family_parenting_val_ls[ind] for ind in range(0,11)}

        # Vacation & Time Off (6 benefits)
        vacation_time_off = 0
        vacation_time_off_ls = ['total_vacation_time_off_benefits','vacation_paid_time_off','sick_days',
                               'paid_holidays','volunteer_time_off','sabbatical', 'bereavement_leave']
        vacation_time_off_val_ls = []
        for ind in range(1,7):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[4]/ul/li["+str(ind)+"]/a").get_attribute('title')
                vacation_time_off +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0
            vacation_time_off_val_ls.append(does_have)
        vacation_time_off_val_ls.insert(0,vacation_time_off) 
        vacation_time_off_dict ={vacation_time_off_ls[ind] : vacation_time_off_val_ls[ind] for ind in range(0,7)}

        # Perk & Discounts (11 benefits)
        perk_discounts = 0
        perk_discounts_ls = ['total_perk_discounts_benefits','employee_discount','free_lunch_or_snacks',
                            'employee_assistance_program','gym_membership','commuter_checks_and_assistance',
                            'pet_friendly_worksplace','mobile_phone_discount','company_car','company_social_events',
                            'travel_concierge','legal_assistance']
        perk_discounts_val_ls = []
        for ind in range(1,12):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[5]/ul/li["+str(ind)+"]/a").get_attribute('title')
                perk_discounts +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0
            perk_discounts_val_ls.append(does_have)
        perk_discounts_val_ls.insert(0,perk_discounts) 
        perk_discounts_dict ={perk_discounts_ls[ind] : perk_discounts_val_ls[ind] for ind in range(0,12)}


        # Professional support (5 benefits)
        professional_support = 0
        professional_support_ls = ['total_professional_support_benefits','diversity_program','job_training',
                                   'professional_development','apprenticeship','tuition_assistance']
        professional_support_val_ls = []
        for ind in range(1,6):
            try:
                temp = driver_benefits.find_element_by_xpath("//*[@id='NodeReplace']/div/div[2]/div[6]/ul/li["+str(ind)+"]/a").get_attribute('title')
                professional_support +=1
                does_have = 1
            except NoSuchElementException:
                does_have = 0
            professional_support_val_ls.append(does_have)
        professional_support_val_ls.insert(0,professional_support)  
        professional_support_dict ={professional_support_ls[ind] : professional_support_val_ls[ind] for ind in range(0,6)}

        #Merge all benefits dictionaries
        benefits_dict = {}
        benefits_dict = {**insurance_health_wellness_dict, 
                **financial_retirement_dict,
                **family_parenting_dict,
                **vacation_time_off_dict,
                **perk_discounts_dict, 
                **professional_support_dict}



        company1 = {'name': name_text, 'industry':indus_text,
                    'office': office_text, 'size': size_text, 'review_counts': reviews_count_text,
                    'salaries_count':salaries_count_text, 'jobs_count' :jobs_count_text, 
                    'interviews_count':interviews_count_text,'benefits_count':benefits_count_text,
                    'website': website, 'company_type': comp_type_text, 'revenue': revenue_text,
                    'head_quarter': hq_text, 'founded':founded_text, 'interview_possitive':interview_pos_text,
                    'interview_negative':interview_neg_text, 'interview_neutral':interview_neut_text, 
                    'interview_difficulty':interview_difficulty_text, 'benefits_score': benefits_score,
                    'rating' : rating_text}   

        company2 = {**company1, **benefits_dict}

        companies_all.append(company2)
        print(name_text,"        ") 
    print('-----------------------------------------------------------end of ',page)
#return companies_all<jupyter_output>-----------------------------------------------------------begin of  346
HERE Technologies         
Veeam Software         
<NAME>         
Adtalem Global Education         
Destination Hotels         
Premier Health         
Babies "R" Us         
Valvoline         
-----------------------------------------------------------end of  346
-----------------------------------------------------------begin of  347
Mission Essential         
St. Peter's Health Partners         
GBMC HealthCare         
Commvault         
Trek Bicycle         
CNH Industrial         
Penn Station         
Consigli Construction         
Cherokee Nation Businesses         
Jaguar Land Rover         
-----------------------------------------------------------end of  347
-----------------------------------------------------------begin of  348
CNH Industrial         
Penn Station         
Memorial Health System (IL)         
AXA XL         
Aggreko         
IDEXX Laboratories         
Vertex Pharmaceuticals  [...]<jupyter_text>Create DataFrame for the result<jupyter_code>companies = pd.DataFrame(companies_all)<jupyter_output><empty_output><jupyter_text>Save the result  to companies.csv<jupyter_code>companies.to_csv('data/companies.csv', index = False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup