<jupyter_start><jupyter_text>Data cleanerThis notebook contains code for cleaning data. Mainly dropping unused columns andremoving unnecessary content from the lyrics.<jupyter_code>import pandas as pd
dataset_list = ["train.csv", "test.csv", "validation.csv"]
# Remove the This Lyrics is NOT for Commercial use portion of lyrics
# NOTE: Don't run this as it has already been done.
# text_to_remove = train["Lyrics"][0][-79:]
# def remove_text(data):
#     """
#     Remove unnecessary text above from the lyrics
#     :param data: dataset
#     :return: cleaned dataset
#     """
#     for i in range(len(data)):
#         data["lyrics"].iloc[i] = data["lyrics"].iloc[i].replace(text_to_remove, "")
def data_cleaner(dataset):
    """
    Cleans the data.
    :param dataset: path to data
    :return: cleaned data
    """

    data = pd.read_csv(dataset)

    # dropping NAN values
    data = data.dropna()

    # Drop columns no longer needed
    # columns_to_drop = ["MSD_sng_id","MSD_track_id"]
    columns_to_drop = ["dzr_sng_id", "valence", "arousal", "artist_name", "track_name"]
    data.drop(columns_to_drop, axis=1, inplace=True)

    # Remove the "This Lyrics is NOT for Commercial use" portion of lyrics
    # remove_text(data)

    return data
for data in dataset_list:
    df = data_cleaner(data)
    df.to_csv(data, index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup