Classification Algorithm Facial Recognition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting and Visualizing the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas.plotting import scatter_matrix\n",
    "data = pd.read_csv('New_Data.csv')\n",
    "\n",
    "X = data.iloc[:,:4]\n",
    "Y = data['Name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "%matplotlib inline\n",
    "from pandas.plotting import scatter_matrix\n",
    "\n",
    "dataFrame = pd.DataFrame(X, columns=[\"Cheek diff\", \"Eye diff\", \"Mouth diff\", \"Nose diff\"])\n",
    " #Gets us the first 5 feature names. \n",
    "scatter_matrix(X, figsize = (10, 10), c = Y, alpha = 0.8, marker = 'O')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_org, X_test_org, y_train, y_test = train_test_split(X, Y, random_state = 0, test_size = 0.2)\n",
    "#Split the data into training and testing sets.\n",
    "\n",
    "scaler = MinMaxScaler() #Scaling the data because sometimes, the data varies a lot(check X.describe())\n",
    "X_train = scaler.fit_transform(X_train_org)\n",
    "X_test = scaler.fit_transform(X_test_org)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_org  #Original without scaling. (Values vary a lot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataFrame = pd.DataFrame(X_train, columns = [\"Cheek diff\", \"Eye diff\", \"Mouth diff\", \"Nose diff\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "columns = X\n",
    "target = Y\n",
    "corr_list = []; #Correlation list- Which features have the biggest role in the accuracy of the model. Higher -> More important.\n",
    "for i in range(0,4):\n",
    "    corr_list.append(np.corrcoef(X_train[:,i], y_train)[0,1])#Get all the rows and the ith column.\n",
    "print(corr_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training KNN for different k's"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "train_score_array = []\n",
    "test_score_array = []\n",
    "\n",
    "for k in range (1, 30):\n",
    "    knn = KNeighborsClassifier(k)\n",
    "    knn.fit(X_train, y_train)\n",
    "    train_score_array.append(knn.score(X_train, y_train))\n",
    "    test_score_array.append(knn.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determining Optimal k Value Through Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_value = 1\n",
    "min = 100\n",
    "k_value2 = 10\n",
    "max = 0\n",
    "d = zip(train_score_array, test_score_array)\n",
    "#What the code below does is find the k-value where the difference between test and training scores is as small as possible.\n",
    "#This k-value is the \"optimal\" k-value.\n",
    "for i, element in enumerate(d):\n",
    "    both = element\n",
    "    value = abs(both[0] - both[1])\n",
    "    value2 = (both[0] + both[1])/2\n",
    "    #print(value)\n",
    "    if(value2 > max and i > 2 and i < 30):\n",
    "        max = value2\n",
    "        k_value2 = i + 1\n",
    "    else:\n",
    "        k_value2 = k_value2\n",
    "    \n",
    "    if(value < min and i > 2 and i < 30):\n",
    "        min = value\n",
    "        k_value = i + 1\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "print(\"The optimal k should be:\",k_value, \"based on the minimum distance between training and testing set accuracies\")\n",
    "print(\"The k value with the highest average accuracy was:\",k_value2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determining Optimal k Value through Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "x_axis = range(1,30) # x_axis values\n",
    "%matplotlib inline\n",
    "#x-values, y-values, Name for legend, color\n",
    "plt.plot(x_axis, train_score_array , label = \"Train Score\", c= \"g\") #Plots a green line\n",
    "plt.plot(x_axis, test_score_array, label = \"Test Score\", c= \"b\")  #Plots a blue line\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search Algorithm for best k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vals = range(1,40)\n",
    "param_grid = {'n_neighbors': vals}\n",
    "print(\"Parameter grid:\\n{}\".format(param_grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(KNeighborsClassifier(), param_grid, cv = 15, return_train_score = True)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best score: {:.4f}\".format(grid_search.best_score_))\n",
    "print(\"Best parameters: {}\".format(grid_search.best_params_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(1)\n",
    "knn.fit(X_train, y_train)\n",
    "print(\"Training Accuracy:\",knn.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", knn.score(X_test, y_test) )#Accuracy of the test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting Custom Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_x = [[0.470437,0.530412,0.486742,0.141006], [0.273302,0.402896,0.928058,0.127295] ]\n",
    "y = knn.predict(new_x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Expected Outputs vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = knn.predict(X_test) #Make predictions for the testing data\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, predictions):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Classifier (SVC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC  \n",
    "svclassifier = SVC(kernel='linear')  \n",
    "svclassifier.fit(X_train, y_train)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\",svclassifier.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", svclassifier.score(X_test, y_test) )#Accuracy of the test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search Algorithm for Optimal Gamma and C Values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declaring parameters grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'C': [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000],\n",
    "              'gamma':[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000], 'kernel': ['linear', 'rbf']}\n",
    "print(\"Parameter grid:\\n{}\".format(param_grid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the Grid Search Algorithm with the parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(SVC(), param_grid, cv = 6, return_train_score = True)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best score: {:.4f}\".format(grid_search.best_score_))\n",
    "print(\"Best parameters: {}\".format(grid_search.best_params_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outpts vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svclassifier = SVC(kernel='rbf', C = 5, gamma = 10)  \n",
    "svclassifier.fit(X_train, y_train)  \n",
    "y_pred = svclassifier.predict(X_test)\n",
    "print(svclassifier.score(X_test, y_test))\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Trees - Gini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "clf_gini = DecisionTreeClassifier(criterion = \"gini\", random_state = 10, max_depth=5, min_samples_leaf=5) \n",
    "# Performing training \n",
    "clf_gini.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\",clf_gini.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", clf_gini.score(X_test, y_test) )#Accuracy of the test.\n",
    "\n",
    "train_score_array = []\n",
    "test_score_array = []\n",
    "\n",
    "for k in range (1, 6):\n",
    "    clf = DecisionTreeClassifier(criterion = \"gini\", random_state = 100, max_depth=k, min_samples_leaf=5)  \n",
    "    clf.fit(X_train, y_train)\n",
    "    train_score_array.append(clf.score(X_train, y_train))\n",
    "    test_score_array.append(clf.score(X_test, y_test))\n",
    "x_axis = range(1,6) # x_axis values\n",
    "%matplotlib inline\n",
    "#x-values, y-values, Name for legend, color\n",
    "plt.plot(x_axis, train_score_array , label = \"Train Score\", c= \"g\") #Plots a green line\n",
    "plt.plot(x_axis, test_score_array, label = \"Test Score\", c= \"b\")  #Plots a blue line\n",
    "plt.xlabel('Maximum depth of trees')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outputs vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf_gini.predict(X_test)\n",
    "\n",
    "values = []\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Trees - Entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_entropy = DecisionTreeClassifier(criterion = \"entropy\", random_state = 100, max_depth = 5, min_samples_leaf = 5) \n",
    "# Performing training \n",
    "clf_entropy.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\",clf_entropy.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", clf_entropy.score(X_test, y_test) )#Accuracy of the test.\n",
    "\n",
    "train_score_array = []\n",
    "test_score_array = []\n",
    "\n",
    "for k in range (1, 6):\n",
    "    clf = DecisionTreeClassifier(criterion = \"entropy\", random_state = 100, max_depth = k, min_samples_leaf = 5) \n",
    "    clf.fit(X_train, y_train)\n",
    "    train_score_array.append(clf.score(X_train, y_train))\n",
    "    test_score_array.append(clf.score(X_test, y_test))\n",
    "x_axis = range(1,6) # x_axis values\n",
    "%matplotlib inline\n",
    "#x-values, y-values, Name for legend, color\n",
    "plt.plot(x_axis, train_score_array , label = \"Train Score\", c= \"g\") #Plots a green line\n",
    "plt.plot(x_axis, test_score_array, label = \"Test Score\", c= \"b\")  #Plots a blue line\n",
    "plt.xlabel('Maximum depth of tree')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outputs vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf_entropy.predict(X_test)\n",
    "\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree - Grid Search Algorithm for best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],\n",
    "              'min_samples_leaf':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 'criterion': ['entropy', 'gini']}\n",
    "print(\"Parameter grid:\\n{}\".format(param_grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(DecisionTreeClassifier(), param_grid, cv = 6, return_train_score = True)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best score: {:.4f}\".format(grid_search.best_score_))\n",
    "print(\"Best parameters: {}\".format(grid_search.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = DecisionTreeClassifier(criterion = \"entropy\", random_state = 10, max_depth = 4, min_samples_leaf = 1) \n",
    "clf.fit(X_train, y_train)\n",
    "print(clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators = 20, random_state = 42)\n",
    "rf.fit(X_train, y_train); #n_estimators is the number of decision trees being used."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\",rf.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", rf.score(X_test, y_test) )#Accuracy of the test.\n",
    "\n",
    "train_score_array = []\n",
    "test_score_array = []\n",
    "\n",
    "for k in range (1, 21):\n",
    "    rf = RandomForestClassifier(n_estimators = k, random_state = 42)\n",
    "    rf.fit(X_train, y_train)\n",
    "    train_score_array.append(rf.score(X_train, y_train))\n",
    "    test_score_array.append(rf.score(X_test, y_test))\n",
    "x_axis = range(1,21) # x_axis values\n",
    "%matplotlib inline\n",
    "#x-values, y-values, Name for legend, color\n",
    "plt.plot(x_axis, train_score_array , label = \"Train Score\", c= \"g\") #Plots a green line\n",
    "plt.plot(x_axis, test_score_array, label = \"Test Score\", c= \"b\")  #Plots a blue line\n",
    "plt.xlabel('# of decision trees')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search Algorithm for best Random Forest parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'n_estimators': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],\n",
    "              'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],\n",
    "              'random_state':[42]}\n",
    "print(\"Parameter grid:\\n{}\".format(param_grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(RandomForestClassifier(), param_grid, cv = 6, return_train_score = True)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best score: {:.4f}\".format(grid_search.best_score_))\n",
    "print(\"Best parameters: {}\".format(grid_search.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(n_estimators = 35, random_state = 42, max_depth = 5, min_samples_leaf = 1) \n",
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(rf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outputs vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test)\n",
    "\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\",lr.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", lr.score(X_test, y_test) )#Accuracy of the test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search Algorithm for Logistic Regression = max_iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'max_iter': [50, 100, 200, 500, 1000, 1500],'multi_class': ['ovr', 'auto']}\n",
    "print(\"Parameter grid:\\n{}\".format(param_grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(LogisticRegression(), param_grid, cv = 6, return_train_score = True)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best score: {:.4f}\".format(grid_search.best_score_))\n",
    "print(\"Best parameters: {}\".format(grid_search.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lreg = LogisticRegression(max_iter = 50, multi_class = 'ovr') \n",
    "lreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lreg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outputs vs Predicted Ouputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lr.predict(X_test)\n",
    "\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB()\n",
    "gnb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training Accuracy:\", gnb.score(X_train, y_train)) #Accuracy of the model when training.\n",
    "print(\"Testing Accuracy:\", gnb.score(X_test, y_test) )#Accuracy of the test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected Outputs vs Predicted Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = gnb.predict(X_test)\n",
    "\n",
    "values = []\n",
    "\n",
    "for i in zip(y_test, y_pred):\n",
    "    values.append(list(i))\n",
    "dataFrame = pd.DataFrame(values, columns = [\"Expected\", \"Predicted\"])\n",
    "dataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "import numpy as np\n",
    "\n",
    "fig = figure(num=None, figsize=(16, 9), dpi=80, facecolor='w', edgecolor='k')\n",
    "\n",
    "objects = ('KNN', 'SVC', 'DT - Gini', 'DT - Entropy', 'Random Forests',  'Gaussian NB', 'Logistic Regression')\n",
    "y_pos = np.arange(len(objects))\n",
    "performance = [90.47,95.24,76.19, 76.19, 80.95, 76.19, 66.66]\n",
    "plt.xticks([0,1,2,3,4,5,6], objects, fontsize=14)\n",
    "plt.yticks([0,20,40,60,80,100], (0,20,40,60,80,100), fontsize=19)\n",
    "    \n",
    "plt.rcParams['xtick.labelsize'] = 20 \n",
    "barlist = plt.bar(y_pos, performance, color = 'g', align='center', alpha=0.5)\n",
    "#plt.x_ticks.font_size(10)\n",
    "#plt.xticks(fontsize=14, rotation=90)\n",
    "barlist[0].set_color('r')\n",
    "barlist[1].set_color('b')\n",
    "barlist[2].set_color('k')\n",
    "barlist[3].set_color('y')\n",
    "barlist[4].set_color('m')\n",
    "barlist[5].set_color('c')\n",
    "\n",
    "plt.ylabel('Accuracy (%)', size = 24)\n",
    "#plt.title('\\n \\nThe accuracy of different classification algorithms on the dataset of facial features after optimization\\n of their hyperparameters through the Grid Search Algorithm.')\n",
    "plt.xlabel(\"Classification Algorithms\", size = 23)\n",
    "plt.suptitle('Accuracy of Different Classification Algorithms After Hyperparameter\\nOptimization Through the Grid Search Algorithm', size = 27)\n",
    "plt.figure(figsize = [16,9])\n",
    "#plt.legend()\n",
    "A = range(7)\n",
    "for xy in zip(A, performance):                                       # <--\n",
    "    plt.annotate('%s%%' % xy[1], xy = xy, textcoords='data') # <--\n",
    "         \n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}