<jupyter_start><jupyter_text>Configuration Settings<jupyter_code># identify the path to the configurations folder
config_path = str(root_path.replace("\\", "/") +"/"+"conf")
args = InputArguments(pathConfFile=config_path)

def prepare_config_path(args) -> dict:
    config_path = f"{args.pathConfFile}/{{}}"
    print("Training configurations filepath: {}".format(config_path.format("config.json")))
    return {"config": config_path.format("config.json")}

file_paths = prepare_config_path(args)

# Read config.py
print("Reading Housing model configuration ........")
config_path = file_paths["config"]
cfg = read_config(config_path=config_path)<jupyter_output>Training configurations filepath: C:/Users/mdetomaso/Desktop/K_Code/custom-ml-pipelines/conf/config.json
Reading Housing model configuration ........<jupyter_text>Training  Collect Housing Data<jupyter_code># Access housing data
housing = DataAccessor(**cfg.input_config).load_housing_data()
housing.head()<jupyter_output>Housing data loaded successfully.<jupyter_text>Train / Test split<jupyter_code># Apply train / test split stratified on median income attribute
train_set, test_set, train_target, test_target = CustomTrainTestSplit(**cfg.testing_config).transform(housing)

# DUMP artitioned data
paths = [cfg.trainset_path, cfg.testset_path, cfg.train_target_path, cfg.test_target_path]
partitions = [train_set, test_set, train_target, test_target]

# save partitioned data
if not os.path.isdir(cfg.partitions_path):
    os.makedirs(cfg.partitions_path)    
for data, path in zip(partitions, paths):
    data.to_csv(path, index=False)
    print("Wrote {}".format(path))<jupyter_output>Stratified Train / Test split.
Train set dimesions: (16512, 11)
Test set dimesions: (4128, 11)
Train target dimesions: (16512,)
Test target dimesions: (4128,)
Wrote data/outputs/partitions/train_set.csv
Wrote data/outputs/partitions/test_set.csv
Wrote data/outputs/partitions/train_target.csv
Wrote data/outputs/partitions/test_target.csv<jupyter_text>Apply standard sklearn Pipeline<jupyter_code># Fit the numerical pipeline to save the scaler across all data
num_pipeline, cat_pipeline, features_pipe, housing_full_pipeline = sklearn_dataflow(config=cfg)
num_pipeline.fit(housing)

# Save fitted scaler
if not os.path.isdir(cfg.artifacts_path):
    os.makedirs(cfg.artifacts_path)
    
scaler = num_pipeline.named_steps['std_scaler']
f = open(cfg.scaler_path, 'wb')
dump(scaler, f)
f.close()
# Load scaler across all housing data
scaler = load(open(cfg.scaler_path, 'rb'))

# Process the training set
num_pipeline, cat_pipeline, features_pipe, housing_full_pipeline = sklearn_dataflow(config=cfg, scaler = scaler)
prepared_trainset = features_pipe.fit_transform(train_set)
prepared_trainset<jupyter_output>Sklearn Housing Prediction Pipeline created.
Numerical missing values imputed.
New numerical attributes created.
Categorical missing values imputed.
Categorical data encoded.<jupyter_text>Train the model and save the best one<jupyter_code># OPTION 1: pass transformed data and ML estimator separately
estimator1 = RandomForestRegressor()
trainer = CustomModelTrainer(estimator1, **cfg.training_config).train(
    X=prepared_trainset, y=train_target
)

final_model_1 = trainer.best_estimator_

# OPTION 2: pass full pipeline with the estimator included
# NOTE: need to modify params_grid consistently
# estimator2 = housing_full_pipeline
# trainer = CustomModelTrainer(estimator2, **cfg.training_config).train(
#    X=train_set, y=train_target
# )

# final_model_2 = trainer.best_estimator_
# SAVE the best estimator
if not os.path.isdir(cfg.artifacts_path):
    os.makedirs(cfg.artifacts_path)
dump(final_model_1, open(cfg.model_path, 'wb'))
print("House model saved.")
# OPTIONAL: Explore used hyperparamters in each run
# cvres = rnd_search.cv_results_
# for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
#    print(np.sqrt(-mean_score), params)
# OPTIONAL: Inspect Pipeline paramters
# for param in housing_full_pipeline.get_params().keys():
#    print(param)<jupyter_output><empty_output><jupyter_text>Testing  Collect Data to predict<jupyter_code># LOAD DATA
test_set = pd.read_csv(cfg.testset_path)
test_target = pd.read_csv(cfg.test_target_path)

print("Test data loaded.")
test_set.head()<jupyter_output>Test data loaded.<jupyter_text>Get feature engineering pipeline<jupyter_code>num_pipe, cat_pipe, feature_pipe, housing_pipeline = sklearn_dataflow(config=cfg, scaler=scaler)
processed_test = feature_pipe.fit_transform(test_set)
processed_test<jupyter_output>Sklearn Housing Prediction Pipeline created.
Numerical missing values imputed.
New numerical attributes created.
Categorical missing values imputed.
Categorical data encoded.<jupyter_text>Make predictions on test data<jupyter_code># LOAD trained model
final_model = load(open(cfg.model_path, 'rb'))
print("Fitted housing pipeline loaded.")
final_predictions = final_model.predict(processed_test)
print("Predictions are: {}".format(final_predictions))<jupyter_output>Fitted housing pipeline loaded.
Predictions are: [487546.82 271099.   230383.01 ... 340668.05 157825.   127998.01]<jupyter_text>Evaluate Predictions<jupyter_code>final_rmse = np.sqrt(mean_squared_error(test_target, final_predictions))
print("Prediction error is {} $".format(round(final_rmse,2)))<jupyter_output>Prediction error is 47220.85 $
StarcoderdataJupyterStructuredCleanDedup