<jupyter_start><jupyter_text>Process json file to DataFrame, which contains the information of all the detectors<jupyter_code>with open('./data/highway_england_sites.json') as file:
    site_info = file.read()
site_info = site_info.split('},{')
site_info_df = []
for c in range(len(site_info)):  # total len: 19022
    info_temp = site_info[c]
    info_temp = [i.split(':') for i in info_temp.split(',')]
    try:
        info_temp[1] = [info_temp[1][0], info_temp[1][1] + info_temp[1][2]]
    except:
        None
    try:
        assert(len(info_temp)==6)
    except:
        temp = ''
        temp_2 = [temp+i[0] for i in info_temp[2:len(info_temp)-4]][0]
        info_temp[1][1] = info_temp[1][1] + ':' + temp_2
        del info_temp[2:len(info_temp)-4]
    for i in range(len(info_temp)):
        for j in range(len(info_temp[i])):
            info_temp[i][j] = info_temp[i][j].strip('"')
    site_info_df.append(pd.Series(pd.DataFrame(info_temp).loc[:, 1]))
    
site_info_df = pd.DataFrame(site_info_df, index = range(len(site_info_df)))
site_info_df.columns = ['Id', 'Name', 'Description', 'Longitude', 'Latitude', 'Status']
site_info_df<jupyter_output><empty_output><jupyter_text>Request csv files  url format: https://webtris.highwaysengland.co.uk/Report/DownloadReport?From=01012021&To=01092021&EnumReportType=1&SiteId=455&EnumReportSubType=1<jupyter_code>root = 'https://webtris.highwaysengland.co.uk/Report/DownloadReport?'
date_from = '01012021'  # ddmmyyyy
date_to = '01092021'  # ddmmyyyy
det_list_M4 = ['M4/2156A', 'M4/2168A', 'M4/2178A', 'M4/2188A', 'M4/2206A', 'M4/2210A', 'M4/2156B', 'M4/2162B', 'M4/2168B', 'M4/2178B', 'M4/2188B', 'M4/2206B', 'M4/2210B']  # selected manually
det_list_M1 = ['M1/2114A', 'M1/2116B', 'M1/2147B', 'M1/2149A', 'M1/2150K', 'M1/2181B', 'M1/2185J', 'M1/2185B', 'M1/2220A', 'M1/2220B', 'M1/2247B', 'M1/2253A', 'M1/2277B', 'M1/2279A', 'M1/2292B', 'M1/2293A', 'M1/2297B', 'M1/2297A', 'M1/2317B', 'M1/2320A', 'M1/2332A', 'M1/2331B', 'M1/2335A']

siteID_M4 = [site_info_df.loc[site_info_df['Description']==i, 'Id'].values[0] for i in det_list_M4]
siteID_M1 = [site_info_df.loc[site_info_df['Description']==i, 'Id'].values[0] for i in det_list_M1]
def get_csv(siteName, siteID):
    url = 'https://webtris.highwaysengland.co.uk/Report/DownloadReport?From=' + \
                date_from + '&To=' + date_to + '&EnumReportType=1&SiteId=' + siteID + '&EnumReportSubType=1'
    urllib.request.urlretrieve(url, './data/%s_%s_%s_%s.csv'%(siteName[:2], siteName[3:], date_from, date_to))
# for i in range(len(det_list_M1)):
#     get_csv(det_list_M1[i], siteID_M1[i])

# for i in range(len(det_list_M4)):
#     get_csv(det_list_M4[i], siteID_M4[i])<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup