<jupyter_start><jupyter_text>Query daily weather data of ChicagoData available at [API](https://darksky.net/dev/docs)<jupyter_code># Get API key (registration needed)
with open("api_key_darksky", 'r') as f:
    api_key = f.readline().strip()

# GPS coordinates of Chicago
lat = '41.836944'
long = '-87.684722'

def get_weather_from_darksky(year, month, day):
    """
    Input:
        :int year
        :int month
        :int day
    Output:
        :dict :dictionary of weather data from web api
    """
    # Get unixtime
    dt = datetime.datetime(year, month, day)
    unixtime = str(int(time.mktime(dt.timetuple())))
    
    # Request darksky.net for data
    r = requests.get('https://api.darksky.net/forecast/'+api_key+'/'+lat+','+long+','+unixtime)

    return json.loads(r.text)    
# Query data for a month and save to file
def save_weather_per_month(year, month):
    """
    Input:
        :int year
        :int month
    Output:
        :void
    """
    month_dict = {}
    
    days = days_in_month(year, month)
    
    for day in range(1, days+1):
        daily_dict = get_weather_from_darksky(year, month, day)
        month_dict[str(year)+'-'+str(month)+'-'+str(day)] = daily_dict
        
    with open("weather_data/"+str(year)+'-'+str(month)+'.json', 'w') as jf:
        json.dump(month_dict, jf)
## Global variable

YEAR = 2013
## Query and save yearly weather data from darksky

for m in range(1, 13):
    print(f'Dealing with month: {m}')
    if not os.path.exists("weather_data/"+str(YEAR)+'-'+str(m)+'.json'):
        print('Collecting data from darksky')
        save_weather_per_month(YEAR, m)
    else:
        print('Previous record found.')<jupyter_output>Dealing with month: 1
Collecting data from darksky
Dealing with month: 2
Collecting data from darksky
Dealing with month: 3
Collecting data from darksky
Dealing with month: 4
Collecting data from darksky
Dealing with month: 5
Collecting data from darksky
Dealing with month: 6
Collecting data from darksky
Dealing with month: 7
Collecting data from darksky
Dealing with month: 8
Collecting data from darksky
Dealing with month: 9
Collecting data from darksky
Dealing with month: 10
Collecting data from darksky
Dealing with month: 11
Collecting data from darksky
Dealing with month: 12
Collecting data from darksky<jupyter_text>Read data from saved file<jupyter_code># drop info that won't be needed
labels_to_drop = ['time', 'summary', 'temperatureMin', 'temperatureMinTime', 'temperatureMax', 'temperatureMaxTime']

# dataframe for yearly weather data
weather_df = pd.DataFrame()

# save data into dataframe
for m in range(1, 13):
    with open("weather_data/"+str(YEAR)+"-"+str(m)+".json", "r") as jf:
        data = json.load(jf)
    for d in range(1, 32):
        try:
            data_dict = data[str(YEAR)+'-'+str(m)+'-'+str(d)]['daily']['data'][0]
            data_series = pd.Series(data_dict).drop(labels=labels_to_drop)
            data_series['day'] = d
            data_series['month'] = m
            data_series['year'] = YEAR
            weather_df = weather_df.append(data_series, ignore_index=True)
        except:
            pass
# Post processing: fill missing data
weather_df['precipType'] = weather_df['precipType'].fillna('None')
weather_df['precipAccumulation'] = weather_df['precipAccumulation'].fillna(0)
weather_df['precipIntensityMaxTime'] = weather_df['precipIntensityMaxTime'].fillna(-1)
# Convert unix time stamp
def convert_unix_timestamp(unixtime):
    """
    Input:
        :float unixtime
    Output:
        :float converted time
    """
    if unixtime == -1:
        return 0
    
    dt = datetime.datetime.fromtimestamp(unixtime)
    
    return dt.hour+dt.minute/60

for tc in ['apparentTemperatureHighTime', 'apparentTemperatureLowTime', 'apparentTemperatureMaxTime', 'apparentTemperatureMinTime',
           'temperatureHighTime', 'temperatureLowTime', 'uvIndexTime', 'windGustTime', 
           'sunsetTime', 'sunriseTime', 'precipIntensityMaxTime']:
    weather_df[tc] = weather_df[tc].apply(lambda x: convert_unix_timestamp(x))
# Add daylight_duration column
weather_df['daylight_duration'] = weather_df['sunsetTime'] - weather_df['sunriseTime']
# One hot encode categorical features
weather_df = pd.get_dummies(weather_df, columns=['icon', 'precipType'])
# Save to csv file
weather_df = weather_df.set_index(['month', 'day', 'year'])
weather_df.to_csv('weather_data/weather_'+str(YEAR)+'_chicago.csv')
weather_df.head()
weather_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 365 entries, (1.0, 1.0, 2013.0) to (12.0, 31.0, 2013.0)
Data columns (total 42 columns):
apparentTemperatureHigh        365 non-null float64
apparentTemperatureHighTime    365 non-null float64
apparentTemperatureLow         365 non-null float64
apparentTemperatureLowTime     365 non-null float64
apparentTemperatureMax         365 non-null float64
apparentTemperatureMaxTime     365 non-null float64
apparentTemperatureMin         365 non-null float64
apparentTemperatureMinTime     365 non-null float64
cloudCover                     365 non-null float64
dewPoint                       365 non-null float64
humidity                       365 non-null float64
moonPhase                      365 non-null float64
precipIntensity                365 non-null float64
precipIntensityMax             365 non-null float64
precipProbability              365 non-null float64
pressure                       365 non-null float64
sunriseTime                  [...]
StarcoderdataJupyterStructuredCleanDedup