<jupyter_start><jupyter_text>ABOUT MISSING VALUES<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.impute import SimpleImputer
data = pd.read_csv("melb_data.csv")
data.shape
data.head(3)
print('persentase nilai null: "{}" \n'.format(data.isnull().sum()/data.shape[0]))
y = data.Price
melb_predictors = data.drop(['Price'], axis = 1)
X = melb_predictors.select_dtypes(exclude = ['object'])
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size = 0.8, test_size = 0.2, random_state = 0)
missing_val_count_by_columns = (X_train.isnull().sum())
print(missing_val_count_by_columns[missing_val_count_by_columns>0])
def score_dataset(X_train, X_valid, y_train, y_valid):
    model = RandomForestRegressor(n_estimators = 10, random_state = 0)
    model.fit(X_train, y_train)
    preds = model.predict(X_valid)
    return mean_absolute_error(y_valid, preds)<jupyter_output><empty_output><jupyter_text>dropping columns<jupyter_code>cols_with_missing = [col for col in X_train.columns if X_train[col].isnull().any()]
reduced_X_train = X_train.drop(cols_with_missing, axis = 1)
reduced_X_valid = X_valid.drop(cols_with_missing, axis = 1)
print("MAE from approach 1 (drop columns with missing values): ")
print(score_dataset(reduced_X_train, reduced_X_valid,  y_train, y_valid))<jupyter_output>MAE from approach 1 (drop columns with missing values): 
183550.22137772635<jupyter_text>imputation<jupyter_code>imputer = SimpleImputer()
imputed_X_train = pd.DataFrame(imputer.fit_transform(X_train))
imputed_X_valid = pd.DataFrame(imputer.transform(X_valid))
imputed_X_train.columns = X_train.columns
imputed_X_valid.columns = X_valid.columns
print("MAE from approach 2 (imputation): ")
print(score_dataset(imputed_X_train, imputed_X_valid,  y_train, y_valid))<jupyter_output>MAE from approach 2 (imputation): 
178166.46269899711<jupyter_text>extension to imputation<jupyter_code>X_train_plus = X_train.copy()
X_valid_plus = X_valid.copy()
for col in cols_with_missing:
    X_train_plus[col + '_was missing'] = X_train_plus[col].isnull()
    X_valid_plus[col + '_was missing'] = X_valid_plus[col].isnull()
    
imputer = SimpleImputer()
imputed_X_train_plus = pd.DataFrame(imputer.fit_transform(X_train_plus))
imputed_X_valid_plus = pd.DataFrame(imputer.transform(X_valid_plus))
imputed_X_train_plus.columns = X_train_plus.columns
imputed_X_valid_plus.columns = X_valid_plus.columns
imputed_X_valid_plus
print("MAE from approach 3 (extention to imputation): ")
print(score_dataset(imputed_X_train_plus, imputed_X_valid_plus,  y_train, y_valid))<jupyter_output>MAE from approach 3 (extention to imputation): 
178927.503183954<jupyter_text>for submission<jupyter_code># Imputation
final_imputer = SimpleImputer(strategy='median')
final_X_train = pd.DataFrame(final_imputer.fit_transform(X_train))
final_X_valid = pd.DataFrame(final_imputer.transform(X_valid))

# Imputation removed column names; put them back
final_X_train.columns = X_train.columns
final_X_valid.columns = X_valid.columns
# Define and fit model
model = RandomForestRegressor(n_estimators=100, random_state=0)
model.fit(final_X_train, y_train)

# Get validation predictions and MAE
preds_valid = model.predict(final_X_valid)
print("MAE (Your approach):")
print(mean_absolute_error(y_valid, preds_valid))
# #untuk data uji
# # Preprocess test data
# final_X_test = pd.DataFrame(final_imputer.transform(X_test))

# # Get test predictions
# preds_test = model.predict(final_X_test)
# # Save test predictions to file
# output = pd.DataFrame({'Id': X_test.index,
#                        'SalePrice': preds_test})
# output.to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>ABOUT CATEGORICAL VARIABLES<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split

# Read the data
data = pd.read_csv('melb_data.csv')

# Separate target from predictors
y = data.Price
X = data.drop(['Price'], axis=1)

# Divide data into training and validation subsets
X_train_full, X_valid_full, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2,
                                                                random_state=0)

# Drop columns with missing values (simplest approach)
cols_with_missing = [col for col in X_train_full.columns if X_train_full[col].isnull().any()] 
X_train_full.drop(cols_with_missing, axis=1, inplace=True)
X_valid_full.drop(cols_with_missing, axis=1, inplace=True)

# "Cardinality" means the number of unique values in a column
# Select categorical columns with relatively low cardinality (convenient but arbitrary)
low_cardinality_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and 
                        X_train_full[cname].dtype == "object"]

# Select numerical columns
numerical_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]

# Keep selected columns only
my_cols = low_cardinality_cols + numerical_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
# Get list of categorical variables
s = (X_train.dtypes == 'object')
object_cols = list(s[s].index)

print("Categorical variables:")
print(object_cols)
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error

# Function for comparing different approaches
def score_dataset(X_train, X_valid, y_train, y_valid):
    model = RandomForestRegressor(n_estimators=100, random_state=0)
    model.fit(X_train, y_train)
    preds = model.predict(X_valid)
    return mean_absolute_error(y_valid, preds)<jupyter_output><empty_output><jupyter_text>dropping categorical variables<jupyter_code>drop_X_train = X_train.select_dtypes(exclude=['object'])
drop_X_valid = X_valid.select_dtypes(exclude=['object'])

print("MAE from Approach 1 (Drop categorical variables):")
print(score_dataset(drop_X_train, drop_X_valid, y_train, y_valid))<jupyter_output>MAE from Approach 1 (Drop categorical variables):
175703.48185157913<jupyter_text>ordinal encoding<jupyter_code>from sklearn.preprocessing import OrdinalEncoder
# Make copy to avoid changing original data 
label_X_train = X_train.copy()
label_X_valid = X_valid.copy()

# Apply ordinal encoder to each column with categorical data
ordinal_encoder = OrdinalEncoder()
label_X_train[object_cols] = ordinal_encoder.fit_transform(X_train[object_cols])
label_X_valid[object_cols] = ordinal_encoder.transform(X_valid[object_cols])

print("MAE from Approach 2 (Ordinal Encoding):") 
print(score_dataset(label_X_train, label_X_valid, y_train, y_valid))<jupyter_output>MAE from Approach 2 (Ordinal Encoding):
165936.40548390493<jupyter_text>one-hot encoding<jupyter_code>from sklearn.preprocessing import OneHotEncoder
# Apply one-hot encoder to each column with categorical data
OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(X_train[object_cols])) #hanya kolom bertipe objek/non-numerik
OH_cols_valid = pd.DataFrame(OH_encoder.transform(X_valid[object_cols]))

# One-hot encoding removed index; put it back
OH_cols_train.index = X_train.index
OH_cols_valid.index = X_valid.index

# Remove categorical columns (will replace with one-hot encoding)
num_X_train = X_train.drop(object_cols, axis=1)
num_X_valid = X_valid.drop(object_cols, axis=1)

#FINAL DATA
# Add one-hot encoded columns to numerical features
OH_X_train = pd.concat([num_X_train, OH_cols_train], axis=1)
OH_X_valid = pd.concat([num_X_valid, OH_cols_valid], axis=1)

print("MAE from Approach 3 (One-Hot Encoding):") 
print(score_dataset(OH_X_train, OH_X_valid, y_train, y_valid))<jupyter_output>MAE from Approach 3 (One-Hot Encoding):
166089.4893009678<jupyter_text>for submission<jupyter_code>#HATI HATI TERHADAP DATA TRAIN DAN VALIDASI TERHADAP KATEGORI VARIABEL YANG SEBAGIAN TIDAK TERMUAT
# # All categorical columns
# object_cols = [col for col in X_train.columns if X_train[col].dtype == "object"]

# # Columns that can be safely ordinal encoded
# good_label_cols = [col for col in object_cols if 
#                    set(X_valid[col]).issubset(set(X_train[col]))]
        
# # Problematic columns that will be dropped from the dataset
# bad_label_cols = list(set(object_cols)-set(good_label_cols))
        
# print('Categorical columns that will be ordinal encoded:', good_label_cols)
# print('\nCategorical columns that will be dropped from the dataset:', bad_label_cols)
# from sklearn.preprocessing import OrdinalEncoder

# # Drop categorical columns that will not be encoded
# label_X_train = X_train.drop(bad_label_cols, axis=1)
# label_X_valid = X_valid.drop(bad_label_cols, axis=1)

# # Apply ordinal encoder 
# ordinal_encoder = OrdinalEncoder()
# label_X_train[good_label_cols] = ordinal_encoder.fit_transform(X_train[good_label_cols])
# label_X_valid[good_label_cols] = ordinal_encoder.transform(X_valid[good_label_cols])  
# # Get number of unique entries in each column with categorical data
# object_nunique = list(map(lambda col: X_train[col].nunique(), object_cols))
# d = dict(zip(object_cols, object_nunique))

# # Print number of unique entries by column, in ascending order
# sorted(d.items(), key=lambda x: x[1])<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup