<jupyter_start><jupyter_text>Cleaning and Profiling<jupyter_code>""" Notebook Setup/Imports
"""

# Project path setup
from pathlib import Path
project_dir = Path(__name__).resolve().parents[1]

# Library import
import pandas as pd
import numpy as np
import pandas_profiling

# Print strings as markdown
from IPython.display import Markdown

# Data variables from previous notebook
dataset = pd.read_pickle(Path.joinpath(project_dir, "data/processed/0_data.pickle"))
data_dictionary = pd.read_pickle(Path.joinpath(project_dir, "data/processed/0_data_dictionary.pickle"))<jupyter_output><empty_output><jupyter_text>Data Type Validation & Cleaning<jupyter_code>""" Get the count for data types in the dataset
"""
d = dataset.dtypes
dataset.get_dtype_counts()
""" Find columns as strings or other objects
"""
object_columns = d[d == 'object'].index.to_list()
print(object_columns)
""" Parse objects to float
"""
for column in object_columns:
    sub = dataset[column].str.replace(",","").str.replace("-","").str.strip().apply(lambda x: np.nan if x == "" else x)
    dataset[column] = sub.astype(float)
""" Check types
"""
dataset.get_dtype_counts()<jupyter_output><empty_output><jupyter_text>Variable Selection<jupyter_code>""" Create a list of rejected variables with > .975 correlation from the profile analysis
"""
profile = pandas_profiling.ProfileReport(dataset, check_recoded=True)
rejected_variables = profile.get_rejected_variables(.975)
""" List of columns to keep
"""
keep_columns = [x for x in dataset.columns if x not in rejected_variables]
""" New dataframe with the columns to keep
"""
dataset_clean = dataset[keep_columns]
""" Set the USDA Model to the percentage 
"""
dataset_clean.drop("USDA Model Count", axis="columns", inplace=True)
dataset_clean.rename({"USDA Model Percent":"USDA Model"},axis="columns",inplace=True)
""" Save the cleaned data
"""
dataset_clean.to_pickle(Path.joinpath(project_dir, "data/processed/1_data.pickle"))
data_dictionary.to_pickle(Path.joinpath(project_dir, "data/processed/1_data_dictionary.pickle"))<jupyter_output><empty_output><jupyter_text>Cleaned Data Profile<jupyter_code>""" Create the data profile object and save it to an HTML file
"""
profile = pandas_profiling.ProfileReport(dataset_clean, check_recoded=True)
profile.to_file(outputfile = Path.joinpath(project_dir,
                                      "docs/_static/profile.html"))<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup