<jupyter_start><jupyter_text>Dealing with nan values<jupyter_code>print(pdf.isnull().sum()) #count of nan values
pdf.fillna(pdf.mean(), inplace=True) #Replacing NaN values with mean
pdf.loc[:, pdf.dtypes == object] = pdf.loc[:, pdf.dtypes == object].fillna('unk') # replacing categorical nan values with 'unk' call for "unknown"
# convert categorical to numerical labels
le = LabelEncoder()
for i in pdf.columns:
    if(pdf[i].dtype == np.object):
        pdf[i] = le.fit_transform(pdf[i])
        
pdf.head()
spark.conf.set("spark.sql.execution.arrow.enabled", "true")
#convert to spark dataframe
bank = spark.createDataFrame(pdf)
bank.show()
#save new data as csv
bank.write.format("csv").option("header", "true").mode("overwrite").save("/FileStore/tables/bank_csv")<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup