#9_Convolution_NN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "#9_Convolution_NN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPwn+LT5W3w9BuAKgDe+Sym",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ariG23498/GrokkingDeepLearning/blob/master/%239_Convolution_NN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zrbtGOdGtgLQ",
        "colab_type": "text"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5znG4dStHfV",
        "colab_type": "code",
        "outputId": "79fdb380-2719-4ce1-85ea-0ef73f9ba65d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%tensorflow_version 2.x\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.datasets import mnist\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from tqdm import tqdm"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPDZhojjc3ES",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.random.seed(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sUv167ubtjFn",
        "colab_type": "text"
      },
      "source": [
        "# The dataset\n",
        "The dataset that is being used here is the MNIST dataset, that is readily available in the `keras` module of `tensorflow`.\n",
        "<br>\n",
        "There are 70,000 images of `hand-written` digits. 60,000 are for training and 10,000 are for testing. The digits range from `0-9`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c6lyy3B4ta_r",
        "colab_type": "code",
        "outputId": "14e44495-a957-4930-af67-3bdbcaaa9d0f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "(x_train,y_train),(x_test,y_test) = mnist.load_data()\n",
        "train_images, train_labels = (x_train.reshape((x_train.shape[0],784))/255,\n",
        "                  y_train)\n",
        "test_images, test_labels = (x_test.reshape((x_test.shape[0],784))/255,\n",
        "                  y_test)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1PXsvi4byUoO",
        "colab_type": "text"
      },
      "source": [
        "## One Hot Encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6zBQVO9yPdw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "one_hot_train = np.zeros((len(train_labels),10))\n",
        "one_hot_test = np.zeros((len(test_labels),10))\n",
        "\n",
        "for i,l in enumerate(train_labels):\n",
        "  one_hot_train[i][l] = 1\n",
        "for i,l in enumerate(test_labels):\n",
        "  one_hot_test[i][l] = 1\n",
        "train_labels = one_hot_train\n",
        "test_labels = one_hot_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kgYzPLrn6ki",
        "colab_type": "code",
        "outputId": "cca3a505-20d0-4c01-da6f-f826784780b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "print('Shape of train images: {}'.format(train_images.shape))\n",
        "print('Shape of train labels: {}'.format(train_labels.shape))\n",
        "\n",
        "print('Shape of test images: {}'.format(test_images.shape))\n",
        "print('Shape of test labels: {}'.format(test_labels.shape))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of train images: (60000, 784)\n",
            "Shape of train labels: (60000, 10)\n",
            "Shape of test images: (10000, 784)\n",
            "Shape of test labels: (10000, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EYIrpMrNy4_c",
        "colab_type": "text"
      },
      "source": [
        "# Activation Functions\n",
        "1. tanh\n",
        "2. softmax"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vbfYpklyxO1e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tanh(x):\n",
        "  return np.tanh(x)\n",
        "\n",
        "def softmax(x):\n",
        "  a = np.exp(x)\n",
        "  return a/np.sum(a,axis=1,keepdims=True)\n",
        "\n",
        "def tanh2deriv(output):\n",
        "  return 1 - (output ** 2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DQe7DgAd02Jv",
        "colab_type": "text"
      },
      "source": [
        "# Configs useful for learning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlLljtWvz1yK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "alpha, iteration = 2, 100\n",
        "pixels_per_image, num_labels = 784, 10\n",
        "batch_size = 128\n",
        "\n",
        "input_rows = 28\n",
        "input_cols = 28\n",
        "\n",
        "kernel_rows = 3\n",
        "kernel_cols = 3\n",
        "num_kernels = 16\n",
        "\n",
        "hidden_size = (input_rows - kernel_rows)*(input_cols - kernel_cols) * num_kernels\n",
        "# hidden_size --> 10000\n",
        "kernels = 0.02*np.random.random((kernel_rows*kernel_cols,num_kernels))-0.01\n",
        "# kernels.shape --> (9, 16)\n",
        "weights_1_2 = 0.2*np.random.random((hidden_size,num_labels))-0.1\n",
        "# weights_1_2.shape --> (10000, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QvluKQt1S_7n",
        "colab_type": "text"
      },
      "source": [
        "# Function to retrieve the subsection of an image"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oEUGJwmES68K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_image_section(layer, row_from, row_to, col_from, col_to):\n",
        "  section = layer[:,row_from:row_to,col_from:col_to]\n",
        "  return section.reshape((-1,1,row_to-row_from,col_to-col_from))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BugbK_0STobE",
        "colab_type": "code",
        "outputId": "60cdf5bf-800a-45dc-8a62-5f33c55c9466",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "for j in range(iteration):\n",
        "  correct_cnt = 0\n",
        "  # Enters each iteration (epoch)\n",
        "  for i in range(len(train_images)//batch_size):\n",
        "    # Enters each batch\n",
        "    batch_start, batch_end = ((i*batch_size),((i+1)*batch_size))\n",
        "    layer_0 = train_images[batch_start:batch_end]\n",
        "    layer_0 = layer_0.reshape((layer_0.shape[0],28,28))\n",
        "    # layer_0.shape --> (128,28,28)\n",
        "\n",
        "    sects = list()\n",
        "    for row_start in range(layer_0.shape[1] - kernel_rows):\n",
        "    # Each row\n",
        "      for col_start in range(layer_0.shape[2] - kernel_cols):\n",
        "      # Each col\n",
        "        sect = get_image_section(layer_0,\n",
        "                                  row_start,\n",
        "                                  row_start + kernel_rows,\n",
        "                                  col_start,\n",
        "                                  col_start + kernel_cols)\n",
        "        sects.append(sect)\n",
        "        # len(sects) --> 625 after one batch\n",
        "        # sect.shape --> (128,1,3,3) --> this means each 3,3 piece of 128 images\n",
        "    \n",
        "    # FORWARD PROPAGATION:\n",
        "    expanded_input = np.concatenate(sects,axis=1)\n",
        "    # expanded_input.shape --> (128,625,3,3)\n",
        "    es = expanded_input.shape\n",
        "    flattened_input = expanded_input.reshape((es[0]*es[1],-1))\n",
        "    # flattened_input.shape --> (80000, 9)\n",
        "\n",
        "    kernel_output = flattened_input.dot(kernels)\n",
        "    layer_1 = tanh(kernel_output.reshape((es[0],-1)))\n",
        "    # layer_1.shape --> (128, 10000)\n",
        "\n",
        "    dropout_mask = np.random.randint(2,size=layer_1.shape)\n",
        "    layer_1 *= dropout_mask * 2\n",
        "    layer_2 = softmax(np.dot(layer_1,weights_1_2))\n",
        "    # layer_2.shape --> (128,10)\n",
        "\n",
        "    # FOR ACCURACY CALCULATION\n",
        "    for k in range(batch_size):\n",
        "      labelset = train_labels[batch_start+k:batch_start+k+1]\n",
        "      _inc = int(np.argmax(layer_2[k:k+1]) == np.argmax(labelset)) # int() is used to convert True False into 1 or 0\n",
        "      correct_cnt += _inc\n",
        "    \n",
        "    # BACK PROPAGATION\n",
        "    layer_2_delta = (train_labels[batch_start:batch_end]-layer_2)/(batch_size*layer_2.shape[0])\n",
        "    layer_1_delta = layer_2_delta.dot(weights_1_2.T)*tanh2deriv(layer_1)\n",
        "    layer_1_delta *= dropout_mask\n",
        "    weights_1_2 += alpha * layer_1.T.dot(layer_2_delta)\n",
        "    l1d_reshape = layer_1_delta.reshape(kernel_output.shape)\n",
        "    # l1d_respahe:  (80000, 16)\n",
        "    k_update = flattened_input.T.dot(l1d_reshape)\n",
        "    # k_update:  (9, 16)\n",
        "    kernels -= alpha * k_update\n",
        "\n",
        "  # TESTING\n",
        "  test_correct_cnt = 0\n",
        "  for i in range(len(test_images)):\n",
        "    layer_0 = test_images[i:i+1]\n",
        "    layer_0 = layer_0.reshape((layer_0.shape[0],28,28))\n",
        "    # layer_0.shape --> (1,28,28)\n",
        "\n",
        "    sects = list()\n",
        "    for row_start in range(layer_0.shape[1] - kernel_rows):\n",
        "      # Sweeps the row\n",
        "      for col_start in range(layer_0.shape[2] - kernel_cols):\n",
        "        # Sweeps the col\n",
        "        sect = get_image_section(layer_0,\n",
        "                                  row_start,\n",
        "                                  row_start + kernel_rows,\n",
        "                                  col_start,\n",
        "                                  col_start + kernel_cols)\n",
        "        sects.append(sect)\n",
        "        # sect.shape --> (1,1,3,3) --> this means each 3,3 piece of 1 test image\n",
        "\n",
        "    # FORWARD PROPAGATION:\n",
        "    expanded_input = np.concatenate(sects,axis=1)\n",
        "    # expanded_input.shape --> (1,625,3,3)\n",
        "    es = expanded_input.shape\n",
        "    flattened_input = expanded_input.reshape((es[0]*es[1],-1))\n",
        "    # flattened_input.shape --> (625, 9)\n",
        "\n",
        "    kernel_output = flattened_input.dot(kernels)\n",
        "    layer_1 = tanh(kernel_output.reshape((es[0],-1)))\n",
        "    # layer_1.shape --> (1, 10000)\n",
        "\n",
        "    dropout_mask = np.random.randint(2,size=layer_1.shape)\n",
        "    layer_1 *= dropout_mask * 2\n",
        "    layer_2 = softmax(np.dot(layer_1,weights_1_2))\n",
        "    # layer_2.shape --> (1,10)\n",
        "\n",
        "    test_correct_cnt += int(np.argmax(layer_2) == np.argmax(test_labels[i:i+1]))\n",
        "  if(j % 10 == 0):\n",
        "    print('Test Acc: {:0.2f}  <--> Train Acc: {:0.2f}'.format((test_correct_cnt/len(test_images)),(correct_cnt/len(train_images))))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Acc: 0.59  <--> Train Acc: 0.23\n",
            "Test Acc: 0.87  <--> Train Acc: 0.86\n",
            "Test Acc: 0.86  <--> Train Acc: 0.85\n",
            "Test Acc: 0.88  <--> Train Acc: 0.89\n",
            "Test Acc: 0.88  <--> Train Acc: 0.89\n",
            "Test Acc: 0.89  <--> Train Acc: 0.89\n",
            "Test Acc: 0.91  <--> Train Acc: 0.91\n",
            "Test Acc: 0.91  <--> Train Acc: 0.91\n",
            "Test Acc: 0.91  <--> Train Acc: 0.91\n",
            "Test Acc: 0.91  <--> Train Acc: 0.91\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eldWHlR2Sfgl",
        "colab_type": "text"
      },
      "source": [
        "# Architecture\n",
        "<img src=\"https://raw.githubusercontent.com/ariG23498/GrokkingDeepLearning/master/assets/CNN.jpg\" height=600 width=450>"
      ]
    }
  ]
}