hyper_parameter_tuning_classification.py

# -*- coding: utf-8 -*-
"""Hyper_parameter_tuning_Classification.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/16LlFZTViSMscShOLpiE1OYW25oF_CNZa

# Hyper-parameter Tunning of Machine Learning (ML) Models

### Code for Classification Problems

#### `Dataset Used:`
MNIST dataset

#### `Machine Learning Algorithm Used:`
* Random Forest (RF) 
* Support Vector Machine (SVM) 
* K-Nearest Neighbor (KNN) 
* Artificial Neural Network (ANN)

#### `Hyper-parameter Tuning Algorithms Used:`
* Grid Search 
* Random Search
* Bayesian Optimization with Gaussian Processes (BO-GP)
* Bayesian Optimization with Tree-structured Parzen Estimator (BO-TPE)

---
"""

# Commented out IPython magic to ensure Python compatibility.
# Importing required libraries
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline
import scipy.stats as stats
from sklearn import datasets
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report,confusion_matrix,accuracy_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC

"""#### Loading MNIST Dataset
The Modified National Institute of Standards and Technology (MNIST) database is a large database of handwritten digits that is commonly used by the people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. It has a training set of 60,000 examples, and a test set of 10,000 examples.It is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image.

It has 1797 record and 64 columns.
For more details about the dataset click here: [Details-1](https://yann.lecun.com/exdb/mnist/), [Details-2](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits/)
"""

# Loading the dataset
X, y = datasets.load_digits(return_X_y=True)
datasets.load_digits()

"""### Baseline Machine Learning Models: Classifier with default Hyper-parameters

### `Random Forest`
"""

# Random Forest (RF) with 3-fold cross validation
RF_clf = RandomForestClassifier()
RF_clf.fit(X,y)
RF_scores = cross_val_score(RF_clf, X, y, cv = 3,scoring = 'accuracy')
print("Accuracy (RF): "+ str(RF_scores.mean()))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_clf = SVC(gamma='scale')
SVM_clf.fit(X,y)
SVM_scores = cross_val_score(SVM_clf, X, y, cv = 3,scoring = 'accuracy')
print("Accuracy (SVM): "+ str(SVM_scores.mean()))

"""### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KNN_clf = KNeighborsClassifier()
KNN_clf.fit(X,y)
KNN_scores = cross_val_score(KNN_clf, X, y, cv = 3,scoring='accuracy')
print("Accuracy (KNN):"+ str(KNN_scores.mean()))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
from keras.models import Sequential, Model
from keras.layers import Dense, Input
from keras.wrappers.scikit_learn import KerasClassifier
from keras.callbacks import EarlyStopping

def ann_model(optimizer = 'sgd',neurons = 32,batch_size = 32,epochs = 50,activation = 'relu',patience = 5,loss = 'categorical_crossentropy'):
    model = Sequential()
    model.add(Dense(neurons, input_shape = (X.shape[1],), activation = activation))
    model.add(Dense(neurons, activation = activation))
    model.add(Dense(10,activation='softmax'))
    model.compile(optimizer = optimizer, loss = loss)
    early_stopping = EarlyStopping(monitor = "loss", patience = patience)
    history = model.fit(X, pd.get_dummies(y).values, batch_size = batch_size, epochs=epochs, callbacks = [early_stopping], verbose=0) 
    return model

ANN_clf = KerasClassifier(build_fn = ann_model, verbose = 0)
ANN_scores = cross_val_score(ANN_clf, X, y, cv = 3,scoring = 'accuracy')
print("Accuracy (ANN):"+ str(ANN_scores.mean()))

"""### Hyper-parameter Tuning Algorithms

### `1] Grid Search`
"""

from sklearn.model_selection import GridSearchCV

"""### `Random Forest`"""

# Random Forest (RF)
RF_params = {
    'n_estimators': [10, 20, 30],
    'max_depth': [15,20,25,30,50],
    "criterion":['gini','entropy']
}
RF_clf = RandomForestClassifier(random_state = 1)
RF_grid = GridSearchCV(RF_clf, RF_params, cv = 3, scoring = 'accuracy')
RF_grid.fit(X, y)
print(RF_grid.best_params_)
print("Accuracy (RF): "+ str(RF_grid.best_score_))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_params = {
    'C': [1, 10, 20, 50, 100],
    "kernel":['linear','poly','rbf','sigmoid']
}
SVM_clf = SVC(gamma='scale')
SVM_grid = GridSearchCV(SVM_clf, SVM_params, cv = 3, scoring = 'accuracy')
SVM_grid.fit(X, y)
print(SVM_grid.best_params_)
print("Accuracy:"+ str(SVM_grid.best_score_))

"""### `K-Nearest Neighbor`"""

#K-Nearest Neighbor (KNN)
KNN_params = { 'n_neighbors': [2, 4, 6, 8] }
KNN_clf = KNeighborsClassifier()
KNN_grid = GridSearchCV(KNN_clf, KNN_params, cv = 3, scoring = 'accuracy')
KNN_grid.fit(X, y)
print(KNN_grid.best_params_)
print("Accuracy:"+ str(KNN_grid.best_score_))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
ANN_params = {
    'optimizer': ['adam','sgd'],
    'activation': ['relu','tanh'],
    'batch_size': [16,32],
    'neurons':[16,32],
    'epochs':[30,50],
    'patience':[3,5]
}
ANN_clf = KerasClassifier(build_fn = ann_model, verbose = 0)
ANN_grid = GridSearchCV(ANN_clf, ANN_params, cv = 3,scoring = 'accuracy')
ANN_grid.fit(X, y)
print(ANN_grid.best_params_)
print("Accuracy (ANN): "+ str(ANN_grid.best_score_))

"""### `2] Random Search`"""

from sklearn.model_selection import RandomizedSearchCV
from random import randrange as sp_randrange
from scipy.stats import randint as sp_randint

"""### `Random Forest`"""

# Random Forest (RF)
RF_params = {
    'n_estimators': sp_randint(10,100),
    'max_depth': sp_randint(5,50),
    "criterion":['gini','entropy']
}
RF_clf = RandomForestClassifier(random_state = 1)
RF_Random = RandomizedSearchCV(RF_clf, param_distributions = RF_params, n_iter = 20,cv = 3,scoring = 'accuracy')
RF_Random.fit(X, y)
print(RF_Random.best_params_)
print("Accuracy (RF):"+ str(RF_Random.best_score_))

"""### `Support Vector Machine`"""

# Support Vector Machine(SVM)
SVM_params = {
    'C': stats.uniform(1,50),
    "kernel":['poly','rbf']
}
SVM_clf = SVC(gamma='scale')
SVM_Random = RandomizedSearchCV(SVM_clf, param_distributions = SVM_params, n_iter = 20,cv = 3,scoring = 'accuracy')
SVM_Random.fit(X, y)
print(SVM_Random.best_params_)
print("Accuracy (SVM): "+ str(SVM_Random.best_score_))

"""### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KNN_params = {'n_neighbors': range(1,20)}
KNN_clf = KNeighborsClassifier()
KNN_Random = RandomizedSearchCV(KNN_clf, param_distributions = KNN_params,n_iter = 10,cv = 3,scoring = 'accuracy')
KNN_Random.fit(X, y)
print(KNN_Random.best_params_)
print("Accuracy (KNN): "+ str(KNN_Random.best_score_))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
ANN_params = {
    'optimizer': ['adam','sgd'],
    'activation': ['relu','tanh'],
    'batch_size': [16,32],
    'neurons':sp_randint(10,100),
    'epochs':[30,50],
    'patience':sp_randint(5,20)
}
ANN_clf = KerasClassifier(build_fn = ann_model, verbose = 0)
ANN_Random = RandomizedSearchCV(ANN_clf, param_distributions = ANN_params, n_iter = 10,cv = 3,scoring = 'accuracy')
ANN_Random.fit(X, y)
print(ANN_Random.best_params_)
print("Accuracy (ANN): "+ str(ANN_Random.best_score_))

"""### `3] Bayesian Optimization with Gaussian Process (BO-GP)`"""

from skopt import Optimizer
from skopt import BayesSearchCV 
from skopt.space import Real, Categorical, Integer

"""### `Random Factor`"""

#Random Forest (RF)
RF_params = {
    'n_estimators': Integer(10,100),
    'max_depth': Integer(5,50),
    "criterion":['gini','entropy']
}
RF_clf = RandomForestClassifier(random_state = 1)
RF_Bayes = BayesSearchCV(RF_clf, RF_params,cv = 3,n_iter = 20, n_jobs = -1,scoring = 'accuracy')
RF_Bayes.fit(X, y)
print(RF_Bayes.best_params_)
print("Accuracy (RF): "+ str(RF_Bayes.best_score_))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
SVM_params = {
    'C': Real(1,50),
    "kernel":['poly','rbf']
}
SVM_clf = SVC(gamma = 'scale')
SVM_Bayes = BayesSearchCV(SVM_clf, SVM_params,cv = 3,n_iter = 20, n_jobs = -1,scoring = 'accuracy')
SVM_Bayes.fit(X, y)
print(SVM_Bayes.best_params_)
print("Accuracy (SVM): "+ str(SVM_Bayes.best_score_))

"""### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
KNN_params = {'n_neighbors': Integer(1,20),}
KNN_clf = KNeighborsClassifier()
KNN_Bayes = BayesSearchCV(KNN_clf, KNN_params,cv = 3,n_iter = 10, n_jobs = -1,scoring = 'accuracy')
KNN_Bayes.fit(X, y)
print(KNN_Bayes.best_params_)
print("Accuracy (KNN): "+ str(KNN_Bayes.best_score_))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
ANN_params = {
    'optimizer': ['adam','sgd'],
    'activation': ['relu','tanh'],
    'batch_size': [16,32],
    'neurons':Integer(10,100),
    'epochs':[30,50],
    'patience':Integer(5,20)
}
ANN_clf = KerasClassifier(build_fn = ann_model, verbose = 0)
ANN_Bayes = BayesSearchCV(ANN_clf, ANN_params,cv = 3,n_iter = 10, scoring = 'accuracy')
ANN_Bayes.fit(X, y)
print(ANN_Bayes.best_params_)
print("Accuracy (ANN): "+ str(ANN_Bayes.best_score_))

"""### `4] Bayesian Optimization with Tree-structured Parzen Estimator (BO-TPE)`"""

from sklearn.model_selection import StratifiedKFold
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials

"""### `Random Forest`"""

# Random Forest (RF)
def RF_fun(params):
    params = {
        'n_estimators': int(params['n_estimators']), 
        'max_features': int(params['max_features']),
        "criterion":str(params['criterion'])
    }
    RF_clf = RandomForestClassifier(**params)
    RF_score = cross_val_score(RF_clf, X, y, cv = StratifiedKFold(n_splits = 3),scoring = 'accuracy').mean()
    return {'loss':-RF_score, 'status': STATUS_OK }

RF_space = {
    'n_estimators': hp.quniform('n_estimators', 10, 100, 1),
    "max_features":hp.quniform('max_features', 1, 32, 1),
    "criterion":hp.choice('criterion',['gini','entropy'])
}

RF_best = fmin(fn = RF_fun, space = RF_space, algo = tpe.suggest, max_evals = 20)
print("Estimated optimum (RF): " +str(RF_best))

"""### `Support Vector Machine`"""

# Support Vector Machine (SVM)
def SVM_fun(params):
    params = {
        'C': abs(float(params['C'])), 
        "kernel":str(params['kernel'])
    }
    SVM_clf = SVC(gamma ='scale', **params)
    SVM_score = cross_val_score(SVM_clf, X, y, cv = StratifiedKFold(n_splits = 3), scoring ='accuracy').mean()
    return {'loss':-SVM_score, 'status': STATUS_OK }
SVM_space = {
    'C': hp.normal('C', 0, 50),
    "kernel":hp.choice('kernel',['poly','rbf'])
}
SVM_best = fmin(fn = SVM_fun, space = SVM_space, algo = tpe.suggest, max_evals = 20)
print("Estimated optimum (SVM): "+str(SVM_best))

"""### `K-Nearest Neighbor`"""

# K-Nearest Neighbor (KNN)
def KNN_fun(params):
    params = {'n_neighbors': abs(int(params['n_neighbors'])) }
    KNN_clf = KNeighborsClassifier(**params)
    KNN_score = cross_val_score(KNN_clf, X, y, cv = StratifiedKFold(n_splits=3), scoring='accuracy').mean()
    return {'loss':-KNN_score, 'status': STATUS_OK }

KNN_space = {'n_neighbors': hp.quniform('n_neighbors', 1, 20, 1)}

KNN_best = fmin(fn = KNN_fun, space = KNN_space, algo = tpe.suggest, max_evals = 10)
print("Estimated optimum (KNN): "+str(KNN_best))

"""### `Artificial Neural Network`"""

# Artificial Neural Network (ANN)
def ANN_fun(params):
    params = {
        "optimizer":str(params['optimizer']),
        "activation":str(params['activation']),
        'batch_size': abs(int(params['batch_size'])),
        'neurons': abs(int(params['neurons'])),
        'epochs': abs(int(params['epochs'])),
        'patience': abs(int(params['patience']))
    }
    ANN_clf = KerasClassifier(build_fn = ann_model,**params, verbose = 0)
    ANN_score = -np.mean(cross_val_score(ANN_clf, X, y, cv=3, scoring = "accuracy"))
    return {'loss':ANN_score, 'status': STATUS_OK }

ANN_space = {
    "optimizer":hp.choice('optimizer',['adam','rmsprop','sgd']),
    "activation":hp.choice('activation',['relu','tanh']),
    'batch_size': hp.quniform('batch_size', 16, 32, 16),
    'neurons': hp.quniform('neurons', 10, 100, 10),
    'epochs': hp.quniform('epochs', 30, 50, 10),
    'patience': hp.quniform('patience', 5, 20, 5),
}

ANN_best = fmin(fn = ANN_fun, space = ANN_space, algo = tpe.suggest, max_evals = 10)
print("Estimated optimum (ANN): "+str(ANN_best))

"""---"""