<jupyter_start><jupyter_text>Cleaning data Code to import and preprocess data.  Convert cell to `code` to do it again.Output: Fichier `final_dataset.csv` ```python importing dataliste_news_mediums=[]liste_vaccine=[]liste_mismatch_index=[]liste_community_risk_index=[]liste_distancing_familiarity=[]liste_news_sources=[]liste_measures_taken=[]liste_visit=[]liste_mask=[]liste_state=["CA","TX","FL","NY","PA","OH","IL","WA","GA","NC","VA","MI","MA","NJ","IN","MO","AZ","TN","CO","MD","MN","WI","OR","KY","OK","AL","SC","NV","UT","LA","KS","AR","IA","CT","NM","ID","MS","WV","ME","NE","MT","NH","RI","DE","HI","SD","AK","ND","VT","WY"]a="http://covidsurvey.mit.edu:5000/query?age=all&gender=all&country=US&us_state="b="&signal=effect_mask"for i in range(0,50):    url=a+liste_state[i]+b    response=requests.get(url).text    liste_mask.append(json.loads(response))c="http://covidsurvey.mit.edu:5000/query?age=all&gender=all&country=US&us_state="d="&signal=news_mediums"for i in range(0,50):    url=a+liste_state[i]+d    response=requests.get(url).text    liste_news_mediums.append(json.loads(response))f="&signal=vaccine_accept"for i in range(0,50):    url=a+liste_state[i]+f    response=requests.get(url).text    liste_vaccine.append(json.loads(response))f="&signal=mismatch_index"for i in range(0,50):    url=a+liste_state[i]+f    response=requests.get(url).text    liste_mismatch_index.append(json.loads(response))f="&signal=community_risk_index"for e in liste_state:    url=a+e+f    response=requests.get(url).text    liste_community_risk_index.append(json.loads(response))f="&signal=distancing_familiarity"for e in liste_state:    url=a+e+f    response=requests.get(url).text    liste_distancing_familiarity.append(json.loads(response))f="&signal=news_sources"for e in liste_state:    url=a+e+f    response=requests.get(url).text    liste_news_sources.append(json.loads(response))f="&signal=measures_taken"for e in liste_state:    url=a+e+f    response=requests.get(url).text    liste_measures_taken.append(json.loads(response))f="&health,retail,restaurants,worship"for e in liste_state:    url=a+e+f    response=requests.get(url).text    liste_visit.append(json.loads(response)) Cleaning datadico={}dico["effect_mask"]=[]dico["vaccine_accept"]=[]dico["mismatch_index"]=[]dico["community_risk_index"]=[]dico["distancing_familiarity"]=[]dico["listen_to_governement_health_autorities"]=[]dico["washing_hands"]=[]dico["wearing_a_face_mask_or_covering"]=[]dico["listen_to_scientists"]=[]dico["read_newspapers"]=[]dico["read_news_online"]=[]dico["isolation"]=[]NaN=Nonefor i in range(len(liste_state)):    if str(liste_mask[i])[2:7]!="error":        dico["effect_mask"].append(liste_mask[i]["effect_mask"]["weighted"]["Extremely effective"])    else:        dico["effect_mask"].append(NaN)    if str(liste_vaccine[i])[2:7]!="error":        dico["vaccine_accept"].append(liste_vaccine[i]["vaccine_accept"]["weighted"]["Yes"])    else:        dico["vaccine_accept"].append(NaN)    if str(liste_mismatch_index[i])[2:7]!="error":        dico["mismatch_index"].append(liste_mismatch_index[i]["mismatch_index"]["weighted"])    else:        dico["mismatch_index"].append(NaN)    if str(liste_community_risk_index[i])[2:7]!="error":        dico["community_risk_index"].append(liste_community_risk_index[i]["community_risk_index"]["weighted"]["Extremely dangerous"])    else:        dico["community_risk_index"].append(NaN)    if str(liste_distancing_familiarity[i])[2:7]!="error":        dico["distancing_familiarity"].append(liste_distancing_familiarity[i]["distancing_familiarity"]["weighted"]["I have heard of it and know what it means"])    else:        dico["distancing_familiarity"].append(NaN)    if str(liste_news_sources[i])[2:7]!="error":        dico["listen_to_governement_health_autorities"].append(liste_news_sources[i]["news_sources"]["government_health_authorities"]["weighted"]["Yes"])        dico["listen_to_scientists"].append(liste_news_sources[i]["news_sources"]["scientists"]["weighted"]["Yes"])    else:        dico["listen_to_governement_health_autorities"].append(NaN)        dico["listen_to_scientists"].append(NaN)    if str(liste_measures_taken[i])[2:7]!="error":        dico["wearing_a_face_mask_or_covering"].append(liste_measures_taken[i]["measures_taken"]["wearing_a_face_mask_or_covering"]["weighted"]["Yes"])        dico["washing_hands"].append(liste_measures_taken[i]["measures_taken"]["washing_hands"]["weighted"]["Yes"])        dico["isolation"].append(liste_measures_taken[i]["measures_taken"]["isolation"]["weighted"]["Yes"])    else:        dico["isolation"].append(NaN)        dico["washing_hands"].append(NaN)        dico["wearing_a_face_mask_or_covering"].append(NaN)    if str(liste_news_mediums[i])[2:7]!="error":        dico["read_newspapers"].append(liste_news_mediums[i]["news_mediums"]["newspapers"]["weighted"]["Yes"])        dico["read_news_online"].append(liste_news_mediums[i]["news_mediums"]["online_sources"]["weighted"]["Yes"])    else:        dico["read_news_online"].append(NaN)        dico["read_newspapers"].append(NaN)data_2=pd.read_csv("data_facebook_survey.csv")data_2.fillna(data_2.mean())corrMatrix = data_2.corr()sn.heatmap(corrMatrix, annot=True)plt.show()import pandas as pddata_number_of_death=pd.read_csv("united_states_covid19_cases_and_deaths_by_state.csv",sep="\n",engine="python",header=None)dico_case_rate_by_states={}for i in range(3,63):    L=data_number_of_death.iloc[i].iloc[0].split(",")    name_state=L[0]    dico_case_rate_by_states[name_state]=float(L[5])states_abreviation=open("states_name.txt")states_abreviation_str=states_abreviation.read()states_abreviation_list=states_abreviation_str.split("\n")dico_states_abreviation={}for e in states_abreviation_list:    G=e.split("\t")    dico_states_abreviation[G[1]]=G[0][:-1]for keys in dico_states_abreviation:    if keys not in liste_state:        print(keys)del dico_states_abreviation["DC"]dico_case_rate_by_states_ordered={}for e in liste_state:    dico_case_rate_by_states_ordered[e]=dico_case_rate_by_states[dico_states_abreviation[e]]print(sum([dico_case_rate_by_states_ordered[keys] for keys in dico_case_rate_by_states_ordered]))dico_case_rate={}dico_case_rate["case_rate"]=[dico_case_rate_by_states_ordered[keys] for keys in dico_case_rate_by_states_ordered]data_case_rate=pd.DataFrame.from_dict(dico_case_rate)final_dataset=pd.merge(data_2,data_case_rate, left_index=True, right_index=True)final_dataset.to_csv("final_dataset.csv",index=False)```<jupyter_code>df = pd.read_csv("final_dataset.csv")
df.head()<jupyter_output><empty_output><jupyter_text>XGBoost model<jupyter_code>X = df.drop(columns=['case_rate'])
y = df.case_rate
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size=0.2)

D_train = xgb.DMatrix(X_train, label=y_train)
D_test = xgb.DMatrix(X_test, label=y_test)
# Define the model
param = {
    'eta': 0.3, 
    'max_depth': 3
}

steps = 20  # The number of training iterations

# Train the model
model = xgb.train(param, D_train, steps)
predictions = model.predict(D_test)
print(f'RMSE: {mean_squared_error(predictions, y_test, squared=False)}')<jupyter_output>RMSE: 1853.5262111357947<jupyter_text>Hyperparameter tuning About 50 minutes to complete. Convert cell to `code` to perform another GridSearch. ```pythonreg = xgb.XGBRegressor()parameters = {    "eta": [0.05, 0.10, 0.15, 0.20, 0.25, 0.30],    "max_depth": [3, 4, 5, 6, 8, 10, 12, 15],    "min_child_weight": [1, 3, 5, 7],    "gamma": [0.0, 0.1, 0.2, 0.3, 0.4],    "colsample_bytree": [0.3, 0.4, 0.5, 0.7]}grid = GridSearchCV(    reg,    parameters,    n_jobs=-1,   -1 means using all processors    scoring="neg_root_mean_squared_error",   RMSE as the chose metric    cv=5,    refit=True,    verbose=3)grid.fit(X, y)model = grid.best_estimator_ Save the optimal modelmodel.save_model('xgboost_params_optimal.json')```<jupyter_code>reg = xgb.XGBRegressor()
reg.load_model('xgboost_params_optimal.json')
predictions = reg.predict(X_test)
print(f'RMSE: {mean_squared_error(predictions, y_test, squared=False)}')<jupyter_output>RMSE: 1569.1415492765361<jupyter_text>**Getting the number of trees:**<jupyter_code>dump_list = reg.get_booster().get_dump()
num_trees = len(dump_list)
num_trees<jupyter_output><empty_output><jupyter_text>Model Explanation with SHAP values<jupyter_code># this takes a minute or two since we are explaining over 30 thousand samples in a model with over a thousand trees
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X)
shap.force_plot(explainer.expected_value, shap_values[0,:], X.columns)
shap.force_plot(explainer.expected_value, shap_values, X.columns)
shap.summary_plot(shap_values, X.columns, plot_type="bar")
shap.summary_plot(shap_values, X)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup