hyper_parameter_tuning_regression.py

# -*- coding: utf-8 -*-
"""Hyper_parameter_tuning_Regression.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1iBpxVtcehX-_Iu-nMEja-sYOgXhv9YRh

# Hyper-parameter Tunning of Machine Learning (ML) Models

### Code for Regression Problems

#### `Dataset Used:`
Boston housing dataset

#### `Machine Learning Algorithm Used:`
* Random Forest (RF) 
* Support Vector Machine (SVM) 
* K-Nearest Neighbor (KNN) 
* Artificial Neural Network (ANN)

#### `Hyper-parameter Tuning Algorithms Used:`
* Grid Search 
* Random Search
* Bayesian Optimization with Gaussian Processes (BO-GP)
* Bayesian Optimization with Tree-structured Parzen Estimator (BO-TPE)

---
"""

# Commented out IPython magic to ensure Python compatibility.
# Importing required libraries
import numpy as np
import pandas as pd 
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline
import scipy.stats as stats
from sklearn import datasets
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR

"""#### Loading Boston Housing Dataset
Boston Housing dataset contains information about different houses in Boston. It contains 506 records with 13 columns. The main goal is to predict the value of prices of the house using the given features.

For more details about the dataset click here: 
[Details-1](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html) ,
[Details-2](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_boston.html)
"""

# Loading dataset
X, y = datasets.load_boston(return_X_y=True)
datasets.load_boston()

print(X.shape) #The data matrix

print(y.shape) #The regression target

"""### Baseline Machine Learning Models: Regressor

###`Random Forest`
"""

# Random Forest (RF) with 3-fold cross validation
RF_clf = RandomForestRegressor()
RF_scores = cross_val_score(RF_clf, X, y, cv = 3, scoring = 'neg_mean_squared_error') 
print("Mean Square Error (RF) :" + str(-RF_scores.mean()))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_clf = SVR(gamma ='scale')
SVM_scores = cross_val_score(SVM_clf, X, y, cv = 3, scoring = 'neg_mean_squared_error')
print("Mean Square Error (SVM) :" + str(-SVM_scores.mean()))

"""### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KN_clf = KNeighborsRegressor()
KN_scores = cross_val_score(KN_clf, X, y, cv = 3,scoring = 'neg_mean_squared_error')
print("Mean Square Error (KNN) :" + str(-KN_scores.mean()))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
from keras.models import Sequential, Model
from keras.layers import Dense, Input
from keras.wrappers.scikit_learn import KerasRegressor
from keras.callbacks import EarlyStopping

def ann_model(optimizer = 'adam', neurons = 32,batch_size = 32, epochs = 50 ,activation = 'relu',patience = 5,loss = 'mse'):
    model = Sequential()
    model.add(Dense(neurons, input_shape = (X.shape[1],), activation = activation))
    model.add(Dense(neurons, activation = activation))
    model.add(Dense(1))
    model.compile(optimizer = optimizer ,loss = loss) 
    early_stopping = EarlyStopping(monitor = "loss", patience = patience)
    history = model.fit(X, y,batch_size = batch_size,epochs = epochs,callbacks = [early_stopping],verbose=0)
    return model

ANN_clf = KerasRegressor(build_fn = ann_model, verbose = 0)
ANN_scores = cross_val_score(ANN_clf, X, y, cv = 3,scoring = 'neg_mean_squared_error')
print("Mean Square Error (ANN):"+ str(-ANN_scores.mean()))

"""### Hyper-parameter Tuning Algorithms

### ` 1] Grid Search`
"""

from sklearn.model_selection import GridSearchCV

"""#### `Random Forest`"""

# Random Forest (RF)
RF_params = {
    'n_estimators': [10, 20, 30],
    'max_depth': [15,20,25,30,50],
}
RF_clf = RandomForestRegressor(random_state = 0)
RF_grid = GridSearchCV(RF_clf, RF_params, cv = 3, scoring = 'neg_mean_squared_error')
RF_grid.fit(X, y)
print(RF_grid.best_params_)
print("Mean Square Error (RF) : "+ str(-RF_grid.best_score_))

"""#### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_params = {
    'C': [1,10, 100,1000],
    'kernel' :['poly','rbf','sigmoid'],
    'epsilon':[0.001, 0.01,0.1,1]
}
SVM_clf = SVR(gamma = 'scale')
SVM_grid = GridSearchCV(SVM_clf, SVM_params, cv = 3, scoring = 'neg_mean_squared_error')
SVM_grid.fit(X, y)
print(SVM_grid.best_params_)
print("Mean Square Error (SVM) :"+ str(-SVM_grid.best_score_))

"""#### `K-Nearest Neighbor`"""

# K-nearest Neighnor (KNN)
KNN_params = {
    'n_neighbors': [2,4,6,8]
}
KNN_clf = KNeighborsRegressor()
KNN_grid = GridSearchCV(KNN_clf, KNN_params, cv=3, scoring='neg_mean_squared_error')
KNN_grid.fit(X, y)
print(KNN_grid.best_params_)
print("Mean Square Error (KNN) :"+ str(-KNN_grid.best_score_))

"""#### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
RF_params = {
    'optimizer': ['adam','rmsprop'],
    'activation': ['relu','tanh'],
    'loss': ['mse','mae'],
    'batch_size': [16,32],
    'neurons':[16,32],
    'epochs':[20,50],
    'patience':[3,5]
}
RF_clf = KerasRegressor(build_fn = ann_model, verbose = 0)
RF_grid = GridSearchCV(RF_clf, RF_params, cv=3,scoring = 'neg_mean_squared_error')
RF_grid.fit(X, y)
print(RF_grid.best_params_)
print("MSE:"+ str(-RF_grid.best_score_))

"""###  `2] Random Search`"""

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint as sp_randint

"""#### `Random Forest`"""

# Random Forest (RF)
RF_params = {
    'n_estimators': sp_randint(10,100),
    'max_depth': sp_randint(5,50),
    "criterion":['mse','mae']
} 
RF_clf = RandomForestRegressor(random_state = 0)
RF_Random = RandomizedSearchCV(RF_clf, param_distributions = RF_params,
                               n_iter = 20 ,cv = 3,scoring = 'neg_mean_squared_error')
RF_Random.fit(X, y)
print(RF_Random.best_params_)
print("Mean Square Error (RF):"+ str(-RF_Random.best_score_))

"""#### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_params = {
    'C': stats.uniform(0,50),
    "kernel":['poly','rbf'],
    "epsilon":stats.uniform(0,1)
}
SVM_clf = SVR(gamma = 'scale')
SVM_Random = RandomizedSearchCV(SVM_clf, param_distributions = SVM_params,
                            n_iter = 20,cv = 3,scoring = 'neg_mean_squared_error')
SVM_Random.fit(X, y)
print(SVM_Random.best_params_)
print("Mean Square Error (SVM) :"+ str(-SVM_Random.best_score_))

"""#### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KNN_params = {
    'n_neighbors': sp_randint(1,20),
}
KNN_clf = KNeighborsRegressor()
KNN_Random = RandomizedSearchCV(KNN_clf, param_distributions = KNN_params,
                            n_iter = 10,cv = 3,scoring = 'neg_mean_squared_error')
KNN_Random.fit(X, y)
print(KNN_Random.best_params_)
print("Mean Square Error (KNN) :"+ str(-KNN_Random.best_score_))

"""#### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
ANN_params = {
    'optimizer': ['adam','rmsprop'],
    'activation': ['relu','tanh'],
    'loss': ['mse','mae'],
    'batch_size': [16,32],
    'neurons':sp_randint(10,100),
    'epochs':[20,50],
    'patience':sp_randint(5,20)
}
ANN_clf = KerasRegressor(build_fn = ann_model, verbose = 0)
ANN_Random = RandomizedSearchCV(ANN_clf, param_distributions = ANN_params,
                                n_iter = 10,cv = 3,scoring = 'neg_mean_squared_error')
ANN_Random.fit(X, y)
print(ANN_Random.best_params_)
print("Mean Square Error (ANN):"+ str(-ANN_Random.best_score_))

"""### `3] Bayesian Optimization with Gaussian Processes (BO-GP)`"""

from skopt import Optimizer
from skopt import BayesSearchCV 
from skopt.space import Real, Categorical, Integer

"""#### `Random Forest`"""

# Random Forest (RF)
RF_params = {
    'n_estimators': Integer(10,100),
    'max_depth': Integer(5,50),
    "criterion":['mse','mae']
}
RF_clf = RandomForestRegressor(random_state = 0)
RF_Bayes = BayesSearchCV(RF_clf, RF_params,cv = 3,n_iter = 20, scoring = 'neg_mean_squared_error') 
RF_Bayes.fit(X, y)
print(RF_Bayes.best_params_)
print("Mean Square Error (RF):"+ str(-RF_Bayes.best_score_))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_params = {
    "kernel":['poly','rbf'],
    'C': Real(1,50),
    'epsilon': Real(0,1)
}
SVM_clf = SVR(gamma='scale')
SVM_Bayes = BayesSearchCV(SVM_clf, SVM_params,cv = 3,n_iter = 20, scoring = 'neg_mean_squared_error')
SVM_Bayes.fit(X, y)
print(SVM_Bayes.best_params_)
print("Mean Square Error (SVM):"+ str(-SVM_Bayes.best_score_))

"""#### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KNN_params = {
    'n_neighbors': Integer(1,20),
}
KNN_clf = KNeighborsRegressor()
KNN_Bayes = BayesSearchCV(KNN_clf, KNN_params,cv = 3,n_iter = 10, scoring = 'neg_mean_squared_error')
KNN_Bayes.fit(X, y)
print(KNN_Bayes.best_params_)
print("Mean Square Error (KNN):"+ str(-KNN_Bayes.best_score_))

"""#### `Artificial Neural Network (ANN)`"""

# Artificial Neural Network (ANN)
ANN_params = {
    'optimizer': ['adam','rmsprop'],
    'activation': ['relu','tanh'],
    'loss': ['mse','mae'],
    'batch_size': [16,32],
    'neurons':Integer(10,100),
    'epochs':[20,50],
    'patience':Integer(5,20)
}
ANN_clf = KerasRegressor(build_fn = ann_model, verbose = 0)
ANN_Bayes = BayesSearchCV(ANN_clf, ANN_params,cv = 3,n_iter = 10, scoring = 'neg_mean_squared_error')
ANN_Bayes.fit(X, y)
print(ANN_Bayes.best_params_)
print("Mean Square Error (ANN):"+ str(-ANN_Bayes.best_score_))

"""### `4] Bayesian Optimization with Tree-structured Parzen Estimator (BO-TPE)`"""

from sklearn.model_selection import cross_val_score, StratifiedKFold
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials

"""#### `Random Forest`"""

# Random Forest (RF)
def RF_fun(params):
    params = {
        'n_estimators': int(params['n_estimators']), 
        'max_depth': int(params['max_depth']),
        "criterion":str(params['criterion'])
    }
    RF_clf = RandomForestRegressor(**params)
    RF_score = -np.mean(cross_val_score(RF_clf, X, y, cv = 3, n_jobs = -1,scoring = "neg_mean_squared_error"))
    return {'loss':RF_score, 'status': STATUS_OK }

RF_space = {
    'n_estimators': hp.quniform('n_estimators', 10, 100, 1),
    'max_depth': hp.quniform('max_depth', 5, 50, 1),
    "criterion":hp.choice('criterion',['mse','mae'])
}

RF_best = fmin(fn = RF_fun, space = RF_space, algo = tpe.suggest, max_evals = 20)
print("Estimated optimum (RF):" +str (RF_best))

"""#### `Support Vector Machine`"""

# Support Vector Machine (SVM)
def SVM_fun(params):
    params = {
        "kernel":str(params['kernel']),
        'C': abs(float(params['C'])), 
        'epsilon': abs(float(params['epsilon'])),
    }
    SVM_clf = SVR(gamma='scale', **params)
    SVM_score = -np.mean(cross_val_score(SVM_clf, X, y, cv = 3, n_jobs = -1, scoring="neg_mean_squared_error"))
    return {'loss':SVM_score, 'status': STATUS_OK }

SVM_space = {
    "kernel":hp.choice('kernel',['poly','rbf']),
    'C': hp.normal('C', 0, 50),
    'epsilon': hp.normal('epsilon', 0, 1),
}

SVM_best = fmin(fn = SVM_fun ,space = SVM_space, algo=tpe.suggest, max_evals = 20)
print("Estimated optimum (SVM):" +str(SVM_best))

"""#### `K-Nearest Neighbor`"""

#K-Nearest Neighbor (KNN)
def KNN_fun(params):
    params = {'n_neighbors': abs(int(params['n_neighbors']))}
    KNN_clf = KNeighborsRegressor(**params)
    KNN_score = -np.mean(cross_val_score(KNN_clf, X, y, cv = 3, n_jobs = -1, scoring = "neg_mean_squared_error"))
    return {'loss':KNN_score, 'status': STATUS_OK }

KNN_space = {'n_neighbors': hp.quniform('n_neighbors', 1, 20, 1),}

KNN_best = fmin(fn = KNN_fun, space = KNN_space,algo = tpe.suggest, max_evals = 10)
print("Estimated optimum (KNN):"+str(KNN_best))

"""#### `Artificial Neural Network`"""

#Artificial Neural Network (ANN)
def ANN_fun(params):
    params = {
        "optimizer":str(params['optimizer']),
        "activation":str(params['activation']),
        "loss":str(params['loss']),
        'batch_size': abs(int(params['batch_size'])),
        'neurons': abs(int(params['neurons'])),
        'epochs': abs(int(params['epochs'])),
        'patience': abs(int(params['patience']))
    }
    ANN_clf = KerasRegressor(build_fn = ann_model,**params, verbose = 0)
    ANN_score = -np.mean(cross_val_score(ANN_clf, X, y, cv = 3, scoring = "neg_mean_squared_error"))
    return {'loss':ANN_score, 'status': STATUS_OK }

ANN_space = {
    "optimizer":hp.choice('optimizer',['adam','rmsprop']),
    "activation":hp.choice('activation',['relu','tanh']),
    "loss":hp.choice('loss',['mse','mae']),
    'batch_size': hp.quniform('batch_size', 16, 32,16),
    'neurons': hp.quniform('neurons', 10, 100,10),
    'epochs': hp.quniform('epochs', 20, 50,20),
    'patience': hp.quniform('patience', 5, 20,5),
}

ANN_best = fmin(fn = ANN_fun, space = ANN_space, algo = tpe.suggest, max_evals = 10)
print("Estimated optimum (ANN): " + str(ANN_best))

"""---"""