cifar10 - CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b46a4b9c",
   "metadata": {},
   "source": [
    "Four different VGG Comvolutional Network are used below to test the effect of dropout on the train set accuracy as well a validation set accuracy. The data set used is cifar-10\n",
    "\n",
    "Models:\n",
    "1. not dropout\n",
    "2. increasing dropout after convolutional layers (0.2 to 0.5) and no dropout in fully connected layers\n",
    "3. increasing dropout after convolutional layers (0.1 to 0.4) and no dropout in fully connected layers\n",
    "4. dropout applied only on fully connected layers \n",
    "5. increasing dropout after convolutional layers and fully connected layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cd6dbde8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........\n",
      "/device:GPU:0\n",
      "........\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 19:23:02.941877: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2021-12-07 19:23:02.941902: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_20 (Conv2D)          (None, 32, 32, 32)        896       \n",
      "                                                                 \n",
      " batch_normalization_20 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_21 (Conv2D)          (None, 32, 32, 32)        9248      \n",
      "                                                                 \n",
      " batch_normalization_21 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_8 (MaxPooling  (None, 16, 16, 32)       0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_22 (Conv2D)          (None, 16, 16, 64)        18496     \n",
      "                                                                 \n",
      " batch_normalization_22 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_23 (Conv2D)          (None, 16, 16, 64)        36928     \n",
      "                                                                 \n",
      " batch_normalization_23 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_9 (MaxPooling  (None, 8, 8, 64)         0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_24 (Conv2D)          (None, 8, 8, 128)         73856     \n",
      "                                                                 \n",
      " batch_normalization_24 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_25 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_25 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_26 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_26 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_10 (MaxPoolin  (None, 4, 4, 128)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " conv2d_27 (Conv2D)          (None, 4, 4, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_27 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_28 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_28 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_29 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_29 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_11 (MaxPoolin  (None, 2, 2, 256)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " flatten_2 (Flatten)         (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_30 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_31 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_8 (Dense)             (None, 10)                10250     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,032,938\n",
      "Trainable params: 4,026,154\n",
      "Non-trainable params: 6,784\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/_3/stf971r13njfhfsz80m7bqwc0000gn/T/ipykernel_1438/218476954.py:130: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
      "  model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 19:23:04.245499: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - ETA: 0s - loss: 1.7288 - accuracy: 0.4325"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 19:23:32.431943: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 32s 38ms/step - loss: 1.7288 - accuracy: 0.4325 - val_loss: 1.5780 - val_accuracy: 0.5388 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 1.1954 - accuracy: 0.6146 - val_loss: 1.5732 - val_accuracy: 0.5647 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 1.0096 - accuracy: 0.6864 - val_loss: 1.1219 - val_accuracy: 0.6645 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.9081 - accuracy: 0.7287 - val_loss: 0.9688 - val_accuracy: 0.7175 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.8433 - accuracy: 0.7509 - val_loss: 0.7871 - val_accuracy: 0.7650 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.7870 - accuracy: 0.7693 - val_loss: 0.9512 - val_accuracy: 0.7281 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.7636 - accuracy: 0.7836 - val_loss: 1.1756 - val_accuracy: 0.6785 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.7122 - accuracy: 0.7976 - val_loss: 0.7665 - val_accuracy: 0.7838 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.6852 - accuracy: 0.8101 - val_loss: 0.7779 - val_accuracy: 0.7844 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.6683 - accuracy: 0.8170 - val_loss: 0.8089 - val_accuracy: 0.7797 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.6458 - accuracy: 0.8266 - val_loss: 0.6577 - val_accuracy: 0.8274 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.6348 - accuracy: 0.8353 - val_loss: 0.6462 - val_accuracy: 0.8343 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6134 - accuracy: 0.8399 - val_loss: 0.6833 - val_accuracy: 0.8223 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.6014 - accuracy: 0.8466 - val_loss: 0.7502 - val_accuracy: 0.8135 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.5890 - accuracy: 0.8514 - val_loss: 0.6362 - val_accuracy: 0.8357 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.5778 - accuracy: 0.8545 - val_loss: 0.6227 - val_accuracy: 0.8441 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.5774 - accuracy: 0.8583 - val_loss: 0.7426 - val_accuracy: 0.8098 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.5611 - accuracy: 0.8632 - val_loss: 0.6627 - val_accuracy: 0.8326 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.5531 - accuracy: 0.8653 - val_loss: 0.6359 - val_accuracy: 0.8439 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.5450 - accuracy: 0.8705 - val_loss: 0.6035 - val_accuracy: 0.8535 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.5445 - accuracy: 0.8707 - val_loss: 0.6523 - val_accuracy: 0.8407 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.4604 - accuracy: 0.8970 - val_loss: 0.7354 - val_accuracy: 0.8501 - lr: 5.0000e-04\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.4337 - accuracy: 0.9046 - val_loss: 0.5308 - val_accuracy: 0.8774 - lr: 5.0000e-04\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.4176 - accuracy: 0.9077 - val_loss: 0.5215 - val_accuracy: 0.8812 - lr: 5.0000e-04\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.4096 - accuracy: 0.9105 - val_loss: 0.5332 - val_accuracy: 0.8734 - lr: 5.0000e-04\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3967 - accuracy: 0.9127 - val_loss: 0.5336 - val_accuracy: 0.8741 - lr: 5.0000e-04\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3902 - accuracy: 0.9148 - val_loss: 0.5108 - val_accuracy: 0.8825 - lr: 5.0000e-04\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3758 - accuracy: 0.9193 - val_loss: 0.5724 - val_accuracy: 0.8771 - lr: 5.0000e-04\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3738 - accuracy: 0.9185 - val_loss: 0.5141 - val_accuracy: 0.8807 - lr: 5.0000e-04\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3685 - accuracy: 0.9210 - val_loss: 0.5527 - val_accuracy: 0.8765 - lr: 5.0000e-04\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3633 - accuracy: 0.9227 - val_loss: 0.5131 - val_accuracy: 0.8813 - lr: 5.0000e-04\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3613 - accuracy: 0.9218 - val_loss: 0.5218 - val_accuracy: 0.8783 - lr: 5.0000e-04\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3527 - accuracy: 0.9252 - val_loss: 0.5101 - val_accuracy: 0.8813 - lr: 5.0000e-04\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3483 - accuracy: 0.9263 - val_loss: 0.5617 - val_accuracy: 0.8742 - lr: 5.0000e-04\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3444 - accuracy: 0.9261 - val_loss: 0.5010 - val_accuracy: 0.8847 - lr: 5.0000e-04\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3409 - accuracy: 0.9274 - val_loss: 0.5698 - val_accuracy: 0.8678 - lr: 5.0000e-04\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3364 - accuracy: 0.9286 - val_loss: 0.5363 - val_accuracy: 0.8798 - lr: 5.0000e-04\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3417 - accuracy: 0.9285 - val_loss: 0.4884 - val_accuracy: 0.8843 - lr: 5.0000e-04\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3285 - accuracy: 0.9327 - val_loss: 0.5324 - val_accuracy: 0.8783 - lr: 5.0000e-04\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.3288 - accuracy: 0.9304 - val_loss: 0.5185 - val_accuracy: 0.8860 - lr: 5.0000e-04\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.3246 - accuracy: 0.9318 - val_loss: 0.5675 - val_accuracy: 0.8717 - lr: 5.0000e-04\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2764 - accuracy: 0.9477 - val_loss: 0.4998 - val_accuracy: 0.8898 - lr: 2.0000e-04\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2620 - accuracy: 0.9524 - val_loss: 0.4995 - val_accuracy: 0.8926 - lr: 2.0000e-04\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2565 - accuracy: 0.9537 - val_loss: 0.5075 - val_accuracy: 0.8890 - lr: 2.0000e-04\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.2463 - accuracy: 0.9581 - val_loss: 0.4929 - val_accuracy: 0.8939 - lr: 2.0000e-04\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2429 - accuracy: 0.9581 - val_loss: 0.5209 - val_accuracy: 0.8882 - lr: 2.0000e-04\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2384 - accuracy: 0.9599 - val_loss: 0.5004 - val_accuracy: 0.8932 - lr: 2.0000e-04\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2339 - accuracy: 0.9603 - val_loss: 0.4775 - val_accuracy: 0.9002 - lr: 2.0000e-04\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2299 - accuracy: 0.9609 - val_loss: 0.5215 - val_accuracy: 0.8885 - lr: 2.0000e-04\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2296 - accuracy: 0.9597 - val_loss: 0.4781 - val_accuracy: 0.8955 - lr: 2.0000e-04\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2231 - accuracy: 0.9622 - val_loss: 0.4698 - val_accuracy: 0.8999 - lr: 2.0000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2217 - accuracy: 0.9626 - val_loss: 0.4858 - val_accuracy: 0.8975 - lr: 2.0000e-04\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2175 - accuracy: 0.9648 - val_loss: 0.4796 - val_accuracy: 0.9003 - lr: 2.0000e-04\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.2149 - accuracy: 0.9648 - val_loss: 0.4964 - val_accuracy: 0.8949 - lr: 2.0000e-04\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2135 - accuracy: 0.9640 - val_loss: 0.5234 - val_accuracy: 0.8926 - lr: 2.0000e-04\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2086 - accuracy: 0.9655 - val_loss: 0.5089 - val_accuracy: 0.8944 - lr: 2.0000e-04\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2097 - accuracy: 0.9649 - val_loss: 0.4983 - val_accuracy: 0.8956 - lr: 2.0000e-04\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2080 - accuracy: 0.9656 - val_loss: 0.4820 - val_accuracy: 0.8972 - lr: 2.0000e-04\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2013 - accuracy: 0.9672 - val_loss: 0.5077 - val_accuracy: 0.8984 - lr: 2.0000e-04\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.2029 - accuracy: 0.9665 - val_loss: 0.4711 - val_accuracy: 0.9010 - lr: 2.0000e-04\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.2007 - accuracy: 0.9664 - val_loss: 0.5004 - val_accuracy: 0.8949 - lr: 2.0000e-04\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1841 - accuracy: 0.9729 - val_loss: 0.4799 - val_accuracy: 0.9013 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1820 - accuracy: 0.9736 - val_loss: 0.4871 - val_accuracy: 0.9019 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1734 - accuracy: 0.9768 - val_loss: 0.4902 - val_accuracy: 0.9023 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1756 - accuracy: 0.9749 - val_loss: 0.4865 - val_accuracy: 0.9025 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1713 - accuracy: 0.9763 - val_loss: 0.4949 - val_accuracy: 0.9012 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1689 - accuracy: 0.9771 - val_loss: 0.4954 - val_accuracy: 0.9030 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1678 - accuracy: 0.9771 - val_loss: 0.4964 - val_accuracy: 0.9037 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1629 - accuracy: 0.9790 - val_loss: 0.5168 - val_accuracy: 0.9000 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1673 - accuracy: 0.9767 - val_loss: 0.5048 - val_accuracy: 0.9027 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1633 - accuracy: 0.9778 - val_loss: 0.5023 - val_accuracy: 0.9010 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1616 - accuracy: 0.9785 - val_loss: 0.5069 - val_accuracy: 0.9007 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1606 - accuracy: 0.9792 - val_loss: 0.5292 - val_accuracy: 0.8987 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1615 - accuracy: 0.9784 - val_loss: 0.5177 - val_accuracy: 0.9019 - lr: 1.0000e-04\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1580 - accuracy: 0.9788 - val_loss: 0.5082 - val_accuracy: 0.9035 - lr: 1.0000e-04\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1590 - accuracy: 0.9777 - val_loss: 0.5117 - val_accuracy: 0.8999 - lr: 1.0000e-04\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1581 - accuracy: 0.9784 - val_loss: 0.4976 - val_accuracy: 0.9042 - lr: 1.0000e-04\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1541 - accuracy: 0.9793 - val_loss: 0.5023 - val_accuracy: 0.9038 - lr: 1.0000e-04\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 49s 63ms/step - loss: 0.1536 - accuracy: 0.9800 - val_loss: 0.5135 - val_accuracy: 0.9007 - lr: 1.0000e-04\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1543 - accuracy: 0.9795 - val_loss: 0.5109 - val_accuracy: 0.9020 - lr: 1.0000e-04\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1539 - accuracy: 0.9796 - val_loss: 0.4990 - val_accuracy: 0.9046 - lr: 1.0000e-04\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1452 - accuracy: 0.9824 - val_loss: 0.5050 - val_accuracy: 0.9083 - lr: 5.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1445 - accuracy: 0.9825 - val_loss: 0.4927 - val_accuracy: 0.9085 - lr: 5.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1415 - accuracy: 0.9833 - val_loss: 0.5055 - val_accuracy: 0.9039 - lr: 5.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1392 - accuracy: 0.9842 - val_loss: 0.5063 - val_accuracy: 0.9044 - lr: 5.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1398 - accuracy: 0.9841 - val_loss: 0.5158 - val_accuracy: 0.9026 - lr: 5.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1392 - accuracy: 0.9842 - val_loss: 0.5136 - val_accuracy: 0.9064 - lr: 5.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1363 - accuracy: 0.9842 - val_loss: 0.5272 - val_accuracy: 0.9018 - lr: 5.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1369 - accuracy: 0.9846 - val_loss: 0.5158 - val_accuracy: 0.9029 - lr: 5.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1347 - accuracy: 0.9847 - val_loss: 0.5109 - val_accuracy: 0.9059 - lr: 5.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1354 - accuracy: 0.9849 - val_loss: 0.5119 - val_accuracy: 0.9029 - lr: 5.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1335 - accuracy: 0.9852 - val_loss: 0.5173 - val_accuracy: 0.9035 - lr: 5.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1323 - accuracy: 0.9854 - val_loss: 0.4992 - val_accuracy: 0.9063 - lr: 5.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 29s 37ms/step - loss: 0.1326 - accuracy: 0.9848 - val_loss: 0.5121 - val_accuracy: 0.9033 - lr: 5.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1319 - accuracy: 0.9856 - val_loss: 0.5315 - val_accuracy: 0.9023 - lr: 5.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1297 - accuracy: 0.9853 - val_loss: 0.5361 - val_accuracy: 0.9019 - lr: 5.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1320 - accuracy: 0.9853 - val_loss: 0.5179 - val_accuracy: 0.9036 - lr: 5.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1306 - accuracy: 0.9855 - val_loss: 0.5193 - val_accuracy: 0.9024 - lr: 5.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1281 - accuracy: 0.9863 - val_loss: 0.5238 - val_accuracy: 0.9051 - lr: 5.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 28s 36ms/step - loss: 0.1321 - accuracy: 0.9852 - val_loss: 0.5251 - val_accuracy: 0.9039 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x315a1a370>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# not dropout used\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.datasets import cifar10\n",
    "from keras import regularizers\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(\"........\")\n",
    "print(tf.test.gpu_device_name())\n",
    "print(\"........\")\n",
    "def lr_schedule(epoch):\n",
    "    lrate = 0.001\n",
    "    if epoch > 80:\n",
    "        lrate = 0.00005\n",
    "    elif epoch > 60:\n",
    "        lrate = 0.0001\n",
    "    elif epoch > 40:\n",
    "        lrate = 0.0002\n",
    "    elif epoch > 20:\n",
    "        lrate = 0.0005        \n",
    "    return lrate\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = x_train / 255, x_test / 255\n",
    "\n",
    "#z-score\n",
    "\n",
    "\n",
    "num_classes = 10\n",
    "y_train = np_utils.to_categorical(y_train,num_classes)\n",
    "y_test = np_utils.to_categorical(y_test,num_classes)\n",
    "\n",
    "weight_decay = 1e-4\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay), input_shape=x_train.shape[1:]))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#data augmentation\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=15,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1,\n",
    "    horizontal_flip=True,\n",
    "    )\n",
    "datagen.fit(x_train)\n",
    "\n",
    "#training\n",
    "batch_size = 64\n",
    "\n",
    "opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e096f468",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........\n",
      "/device:GPU:0\n",
      "........\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 20:12:31.314718: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2021-12-07 20:12:31.314744: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_40 (Conv2D)          (None, 32, 32, 32)        896       \n",
      "                                                                 \n",
      " batch_normalization_44 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_41 (Conv2D)          (None, 32, 32, 32)        9248      \n",
      "                                                                 \n",
      " batch_normalization_45 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_16 (MaxPoolin  (None, 16, 16, 32)       0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_4 (Dropout)         (None, 16, 16, 32)        0         \n",
      "                                                                 \n",
      " conv2d_42 (Conv2D)          (None, 16, 16, 64)        18496     \n",
      "                                                                 \n",
      " batch_normalization_46 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_43 (Conv2D)          (None, 16, 16, 64)        36928     \n",
      "                                                                 \n",
      " batch_normalization_47 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_17 (MaxPoolin  (None, 8, 8, 64)         0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_5 (Dropout)         (None, 8, 8, 64)          0         \n",
      "                                                                 \n",
      " conv2d_44 (Conv2D)          (None, 8, 8, 128)         73856     \n",
      "                                                                 \n",
      " batch_normalization_48 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_45 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_49 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_46 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_50 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_18 (MaxPoolin  (None, 4, 4, 128)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_6 (Dropout)         (None, 4, 4, 128)         0         \n",
      "                                                                 \n",
      " conv2d_47 (Conv2D)          (None, 4, 4, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_51 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_48 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_52 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_49 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_53 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_19 (MaxPoolin  (None, 2, 2, 256)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_7 (Dropout)         (None, 2, 2, 256)         0         \n",
      "                                                                 \n",
      " flatten_4 (Flatten)         (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_12 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_54 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_13 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_55 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_14 (Dense)            (None, 10)                10250     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,032,938\n",
      "Trainable params: 4,026,154\n",
      "Non-trainable params: 6,784\n",
      "_________________________________________________________________\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 20:12:32.605596: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "780/781 [============================>.] - ETA: 0s - loss: 1.8665 - accuracy: 0.3876"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 20:13:02.653952: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 34s 39ms/step - loss: 1.8664 - accuracy: 0.3876 - val_loss: 2.3577 - val_accuracy: 0.3822 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 1.3507 - accuracy: 0.5582 - val_loss: 1.4362 - val_accuracy: 0.5699 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 1.1528 - accuracy: 0.6306 - val_loss: 1.2486 - val_accuracy: 0.6342 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 1.0480 - accuracy: 0.6730 - val_loss: 1.0211 - val_accuracy: 0.6867 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.9801 - accuracy: 0.7016 - val_loss: 1.1208 - val_accuracy: 0.6784 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.9253 - accuracy: 0.7258 - val_loss: 0.9619 - val_accuracy: 0.7060 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.8799 - accuracy: 0.7412 - val_loss: 0.8231 - val_accuracy: 0.7628 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.8503 - accuracy: 0.7540 - val_loss: 0.8715 - val_accuracy: 0.7436 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.8285 - accuracy: 0.7618 - val_loss: 0.7609 - val_accuracy: 0.7882 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.8061 - accuracy: 0.7718 - val_loss: 1.4084 - val_accuracy: 0.6810 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7865 - accuracy: 0.7802 - val_loss: 0.8336 - val_accuracy: 0.7714 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7667 - accuracy: 0.7869 - val_loss: 0.9655 - val_accuracy: 0.7368 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7576 - accuracy: 0.7914 - val_loss: 0.7849 - val_accuracy: 0.7785 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7399 - accuracy: 0.7985 - val_loss: 0.7742 - val_accuracy: 0.7944 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7325 - accuracy: 0.8025 - val_loss: 0.7341 - val_accuracy: 0.8058 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7223 - accuracy: 0.8058 - val_loss: 0.7211 - val_accuracy: 0.8071 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7098 - accuracy: 0.8100 - val_loss: 0.6726 - val_accuracy: 0.8263 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7010 - accuracy: 0.8156 - val_loss: 0.6746 - val_accuracy: 0.8281 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.6953 - accuracy: 0.8175 - val_loss: 0.6957 - val_accuracy: 0.8208 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6866 - accuracy: 0.8211 - val_loss: 0.7823 - val_accuracy: 0.7967 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6827 - accuracy: 0.8221 - val_loss: 0.7212 - val_accuracy: 0.8171 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6167 - accuracy: 0.8451 - val_loss: 0.5801 - val_accuracy: 0.8591 - lr: 5.0000e-04\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5831 - accuracy: 0.8533 - val_loss: 0.6302 - val_accuracy: 0.8381 - lr: 5.0000e-04\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5736 - accuracy: 0.8553 - val_loss: 0.5609 - val_accuracy: 0.8609 - lr: 5.0000e-04\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5632 - accuracy: 0.8577 - val_loss: 0.5386 - val_accuracy: 0.8715 - lr: 5.0000e-04\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5548 - accuracy: 0.8608 - val_loss: 0.5490 - val_accuracy: 0.8614 - lr: 5.0000e-04\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5409 - accuracy: 0.8635 - val_loss: 0.5506 - val_accuracy: 0.8628 - lr: 5.0000e-04\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5341 - accuracy: 0.8646 - val_loss: 0.5394 - val_accuracy: 0.8647 - lr: 5.0000e-04\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5322 - accuracy: 0.8655 - val_loss: 0.5624 - val_accuracy: 0.8610 - lr: 5.0000e-04\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 314s 402ms/step - loss: 0.5192 - accuracy: 0.8680 - val_loss: 0.5443 - val_accuracy: 0.8638 - lr: 5.0000e-04\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5183 - accuracy: 0.8714 - val_loss: 0.5396 - val_accuracy: 0.8663 - lr: 5.0000e-04\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5224 - accuracy: 0.8695 - val_loss: 0.5705 - val_accuracy: 0.8521 - lr: 5.0000e-04\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.5104 - accuracy: 0.8724 - val_loss: 0.5480 - val_accuracy: 0.8655 - lr: 5.0000e-04\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5058 - accuracy: 0.8729 - val_loss: 0.5103 - val_accuracy: 0.8721 - lr: 5.0000e-04\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5001 - accuracy: 0.8749 - val_loss: 0.7150 - val_accuracy: 0.8473 - lr: 5.0000e-04\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4971 - accuracy: 0.8756 - val_loss: 0.5535 - val_accuracy: 0.8632 - lr: 5.0000e-04\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5011 - accuracy: 0.8745 - val_loss: 0.5275 - val_accuracy: 0.8705 - lr: 5.0000e-04\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4934 - accuracy: 0.8755 - val_loss: 0.5616 - val_accuracy: 0.8576 - lr: 5.0000e-04\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4888 - accuracy: 0.8772 - val_loss: 0.5206 - val_accuracy: 0.8750 - lr: 5.0000e-04\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4868 - accuracy: 0.8771 - val_loss: 0.5516 - val_accuracy: 0.8597 - lr: 5.0000e-04\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4778 - accuracy: 0.8796 - val_loss: 0.5801 - val_accuracy: 0.8592 - lr: 5.0000e-04\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4412 - accuracy: 0.8941 - val_loss: 0.4777 - val_accuracy: 0.8868 - lr: 2.0000e-04\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4237 - accuracy: 0.8987 - val_loss: 0.4953 - val_accuracy: 0.8803 - lr: 2.0000e-04\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4151 - accuracy: 0.9002 - val_loss: 0.4947 - val_accuracy: 0.8825 - lr: 2.0000e-04\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4110 - accuracy: 0.9009 - val_loss: 0.5079 - val_accuracy: 0.8803 - lr: 2.0000e-04\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4033 - accuracy: 0.9047 - val_loss: 0.4558 - val_accuracy: 0.8930 - lr: 2.0000e-04\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4015 - accuracy: 0.9038 - val_loss: 0.4915 - val_accuracy: 0.8827 - lr: 2.0000e-04\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3972 - accuracy: 0.9051 - val_loss: 0.4924 - val_accuracy: 0.8833 - lr: 2.0000e-04\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3987 - accuracy: 0.9027 - val_loss: 0.4708 - val_accuracy: 0.8862 - lr: 2.0000e-04\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3879 - accuracy: 0.9068 - val_loss: 0.4935 - val_accuracy: 0.8798 - lr: 2.0000e-04\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3852 - accuracy: 0.9087 - val_loss: 0.4793 - val_accuracy: 0.8876 - lr: 2.0000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3821 - accuracy: 0.9076 - val_loss: 0.4458 - val_accuracy: 0.8939 - lr: 2.0000e-04\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3783 - accuracy: 0.9093 - val_loss: 0.4577 - val_accuracy: 0.8886 - lr: 2.0000e-04\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3771 - accuracy: 0.9090 - val_loss: 0.4502 - val_accuracy: 0.8924 - lr: 2.0000e-04\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3771 - accuracy: 0.9094 - val_loss: 0.4491 - val_accuracy: 0.8925 - lr: 2.0000e-04\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3674 - accuracy: 0.9122 - val_loss: 0.4587 - val_accuracy: 0.8913 - lr: 2.0000e-04\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3740 - accuracy: 0.9097 - val_loss: 0.4547 - val_accuracy: 0.8873 - lr: 2.0000e-04\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.3652 - accuracy: 0.9131 - val_loss: 0.4739 - val_accuracy: 0.8871 - lr: 2.0000e-04\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3670 - accuracy: 0.9110 - val_loss: 0.4563 - val_accuracy: 0.8900 - lr: 2.0000e-04\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3589 - accuracy: 0.9141 - val_loss: 0.4779 - val_accuracy: 0.8856 - lr: 2.0000e-04\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3608 - accuracy: 0.9134 - val_loss: 0.4564 - val_accuracy: 0.8913 - lr: 2.0000e-04\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3407 - accuracy: 0.9189 - val_loss: 0.4447 - val_accuracy: 0.8965 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3403 - accuracy: 0.9196 - val_loss: 0.4602 - val_accuracy: 0.8924 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3369 - accuracy: 0.9214 - val_loss: 0.4428 - val_accuracy: 0.8952 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3305 - accuracy: 0.9226 - val_loss: 0.4574 - val_accuracy: 0.8953 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3326 - accuracy: 0.9211 - val_loss: 0.4826 - val_accuracy: 0.8958 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3328 - accuracy: 0.9203 - val_loss: 0.4343 - val_accuracy: 0.8995 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3240 - accuracy: 0.9249 - val_loss: 0.4587 - val_accuracy: 0.8903 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3225 - accuracy: 0.9241 - val_loss: 0.4408 - val_accuracy: 0.8954 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3238 - accuracy: 0.9241 - val_loss: 0.4470 - val_accuracy: 0.8929 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3203 - accuracy: 0.9246 - val_loss: 0.4461 - val_accuracy: 0.8953 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3216 - accuracy: 0.9232 - val_loss: 0.4539 - val_accuracy: 0.8937 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3171 - accuracy: 0.9266 - val_loss: 0.4453 - val_accuracy: 0.8934 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.3184 - accuracy: 0.9256 - val_loss: 0.4276 - val_accuracy: 0.8980 - lr: 1.0000e-04\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3134 - accuracy: 0.9276 - val_loss: 0.4735 - val_accuracy: 0.8860 - lr: 1.0000e-04\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3148 - accuracy: 0.9274 - val_loss: 0.4521 - val_accuracy: 0.8956 - lr: 1.0000e-04\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3128 - accuracy: 0.9262 - val_loss: 0.4352 - val_accuracy: 0.8977 - lr: 1.0000e-04\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3123 - accuracy: 0.9264 - val_loss: 0.4542 - val_accuracy: 0.8932 - lr: 1.0000e-04\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3135 - accuracy: 0.9268 - val_loss: 0.4263 - val_accuracy: 0.9010 - lr: 1.0000e-04\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3057 - accuracy: 0.9292 - val_loss: 0.4454 - val_accuracy: 0.8917 - lr: 1.0000e-04\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3085 - accuracy: 0.9284 - val_loss: 0.4184 - val_accuracy: 0.8995 - lr: 1.0000e-04\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3029 - accuracy: 0.9295 - val_loss: 0.4276 - val_accuracy: 0.8977 - lr: 5.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2914 - accuracy: 0.9330 - val_loss: 0.4448 - val_accuracy: 0.8957 - lr: 5.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2947 - accuracy: 0.9330 - val_loss: 0.4436 - val_accuracy: 0.8989 - lr: 5.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2964 - accuracy: 0.9308 - val_loss: 0.4423 - val_accuracy: 0.9009 - lr: 5.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2946 - accuracy: 0.9329 - val_loss: 0.4301 - val_accuracy: 0.9010 - lr: 5.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2912 - accuracy: 0.9328 - val_loss: 0.4332 - val_accuracy: 0.8982 - lr: 5.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2908 - accuracy: 0.9333 - val_loss: 0.4446 - val_accuracy: 0.8960 - lr: 5.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2888 - accuracy: 0.9338 - val_loss: 0.4404 - val_accuracy: 0.8983 - lr: 5.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2862 - accuracy: 0.9345 - val_loss: 0.4476 - val_accuracy: 0.8973 - lr: 5.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2902 - accuracy: 0.9331 - val_loss: 0.4520 - val_accuracy: 0.8968 - lr: 5.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2865 - accuracy: 0.9344 - val_loss: 0.4543 - val_accuracy: 0.8991 - lr: 5.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2870 - accuracy: 0.9346 - val_loss: 0.4740 - val_accuracy: 0.8954 - lr: 5.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2836 - accuracy: 0.9331 - val_loss: 0.4902 - val_accuracy: 0.8968 - lr: 5.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2848 - accuracy: 0.9352 - val_loss: 0.4774 - val_accuracy: 0.8983 - lr: 5.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2863 - accuracy: 0.9337 - val_loss: 0.5210 - val_accuracy: 0.8947 - lr: 5.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2803 - accuracy: 0.9371 - val_loss: 0.5041 - val_accuracy: 0.8988 - lr: 5.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2819 - accuracy: 0.9351 - val_loss: 0.5010 - val_accuracy: 0.8969 - lr: 5.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2808 - accuracy: 0.9357 - val_loss: 0.4626 - val_accuracy: 0.8983 - lr: 5.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2815 - accuracy: 0.9344 - val_loss: 0.4900 - val_accuracy: 0.8999 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2a852be80>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# increasing dropout after convolutional layers (0.2 to 0.5) and no dropout in fully connected layers\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.datasets import cifar10\n",
    "from keras import regularizers\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(\"........\")\n",
    "print(tf.test.gpu_device_name())\n",
    "print(\"........\")\n",
    "def lr_schedule(epoch):\n",
    "    lrate = 0.001\n",
    "    if epoch > 80:\n",
    "        lrate = 0.00005\n",
    "    elif epoch > 60:\n",
    "        lrate = 0.0001\n",
    "    elif epoch > 40:\n",
    "        lrate = 0.0002\n",
    "    elif epoch > 20:\n",
    "        lrate = 0.0005        \n",
    "    return lrate\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = x_train / 255, x_test / 255\n",
    "\n",
    "#z-score\n",
    "\n",
    "\n",
    "num_classes = 10\n",
    "y_train = np_utils.to_categorical(y_train,num_classes)\n",
    "y_test = np_utils.to_categorical(y_test,num_classes)\n",
    "\n",
    "weight_decay = 1e-4\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay), input_shape=x_train.shape[1:]))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.2))\n",
    "\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.3))\n",
    "\n",
    "\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.4))\n",
    "\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.5))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#data augmentation\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=15,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1,\n",
    "    horizontal_flip=True,\n",
    "    )\n",
    "datagen.fit(x_train)\n",
    "\n",
    "#training\n",
    "batch_size = 64\n",
    "\n",
    "opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "375b242c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........\n",
      "/device:GPU:0\n",
      "........\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 21:08:31.727725: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2021-12-07 21:08:31.727753: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_50 (Conv2D)          (None, 32, 32, 32)        896       \n",
      "                                                                 \n",
      " batch_normalization_56 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_51 (Conv2D)          (None, 32, 32, 32)        9248      \n",
      "                                                                 \n",
      " batch_normalization_57 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_20 (MaxPoolin  (None, 16, 16, 32)       0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_8 (Dropout)         (None, 16, 16, 32)        0         \n",
      "                                                                 \n",
      " conv2d_52 (Conv2D)          (None, 16, 16, 64)        18496     \n",
      "                                                                 \n",
      " batch_normalization_58 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_53 (Conv2D)          (None, 16, 16, 64)        36928     \n",
      "                                                                 \n",
      " batch_normalization_59 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_21 (MaxPoolin  (None, 8, 8, 64)         0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_9 (Dropout)         (None, 8, 8, 64)          0         \n",
      "                                                                 \n",
      " conv2d_54 (Conv2D)          (None, 8, 8, 128)         73856     \n",
      "                                                                 \n",
      " batch_normalization_60 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_55 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_61 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_56 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_62 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_22 (MaxPoolin  (None, 4, 4, 128)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_10 (Dropout)        (None, 4, 4, 128)         0         \n",
      "                                                                 \n",
      " conv2d_57 (Conv2D)          (None, 4, 4, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_63 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_58 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_64 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_59 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_65 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_23 (MaxPoolin  (None, 2, 2, 256)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " dropout_11 (Dropout)        (None, 2, 2, 256)         0         \n",
      "                                                                 \n",
      " flatten_5 (Flatten)         (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_15 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_66 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_16 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_67 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_17 (Dense)            (None, 10)                10250     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,032,938\n",
      "Trainable params: 4,026,154\n",
      "Non-trainable params: 6,784\n",
      "_________________________________________________________________\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 21:08:33.225656: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - ETA: 0s - loss: 1.8052 - accuracy: 0.4053"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 21:09:04.638483: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 36s 41ms/step - loss: 1.8052 - accuracy: 0.4053 - val_loss: 1.5611 - val_accuracy: 0.5443 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 33s 42ms/step - loss: 1.2842 - accuracy: 0.5793 - val_loss: 1.4839 - val_accuracy: 0.5363 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 1.0890 - accuracy: 0.6542 - val_loss: 1.0696 - val_accuracy: 0.6542 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 33s 42ms/step - loss: 0.9833 - accuracy: 0.6981 - val_loss: 1.0072 - val_accuracy: 0.6951 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.9170 - accuracy: 0.7225 - val_loss: 0.9989 - val_accuracy: 0.7066 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8574 - accuracy: 0.7456 - val_loss: 0.9872 - val_accuracy: 0.7319 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8242 - accuracy: 0.7589 - val_loss: 0.8697 - val_accuracy: 0.7483 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7952 - accuracy: 0.7714 - val_loss: 0.9586 - val_accuracy: 0.7420 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7721 - accuracy: 0.7819 - val_loss: 0.7500 - val_accuracy: 0.7924 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7493 - accuracy: 0.7933 - val_loss: 0.7767 - val_accuracy: 0.7885 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.7260 - accuracy: 0.7995 - val_loss: 0.7572 - val_accuracy: 0.7959 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7139 - accuracy: 0.8045 - val_loss: 0.8786 - val_accuracy: 0.7942 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6968 - accuracy: 0.8129 - val_loss: 0.9376 - val_accuracy: 0.7385 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.6832 - accuracy: 0.8209 - val_loss: 0.6960 - val_accuracy: 0.8263 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6774 - accuracy: 0.8225 - val_loss: 0.7709 - val_accuracy: 0.7987 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6671 - accuracy: 0.8276 - val_loss: 0.6241 - val_accuracy: 0.8423 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6553 - accuracy: 0.8311 - val_loss: 0.7231 - val_accuracy: 0.8112 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6525 - accuracy: 0.8325 - val_loss: 0.7519 - val_accuracy: 0.8095 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6391 - accuracy: 0.8387 - val_loss: 0.6836 - val_accuracy: 0.8287 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6340 - accuracy: 0.8408 - val_loss: 0.7131 - val_accuracy: 0.8234 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6261 - accuracy: 0.8437 - val_loss: 0.7311 - val_accuracy: 0.8180 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5527 - accuracy: 0.8685 - val_loss: 0.5979 - val_accuracy: 0.8543 - lr: 5.0000e-04\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5287 - accuracy: 0.8745 - val_loss: 0.5661 - val_accuracy: 0.8645 - lr: 5.0000e-04\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5139 - accuracy: 0.8771 - val_loss: 0.5410 - val_accuracy: 0.8729 - lr: 5.0000e-04\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5067 - accuracy: 0.8777 - val_loss: 0.5339 - val_accuracy: 0.8719 - lr: 5.0000e-04\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4951 - accuracy: 0.8801 - val_loss: 0.6084 - val_accuracy: 0.8452 - lr: 5.0000e-04\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4866 - accuracy: 0.8843 - val_loss: 0.4970 - val_accuracy: 0.8830 - lr: 5.0000e-04\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4761 - accuracy: 0.8865 - val_loss: 0.5199 - val_accuracy: 0.8783 - lr: 5.0000e-04\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4736 - accuracy: 0.8875 - val_loss: 0.5293 - val_accuracy: 0.8752 - lr: 5.0000e-04\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4598 - accuracy: 0.8900 - val_loss: 0.4887 - val_accuracy: 0.8847 - lr: 5.0000e-04\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4646 - accuracy: 0.8894 - val_loss: 0.5069 - val_accuracy: 0.8754 - lr: 5.0000e-04\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4562 - accuracy: 0.8920 - val_loss: 0.5201 - val_accuracy: 0.8750 - lr: 5.0000e-04\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4569 - accuracy: 0.8896 - val_loss: 0.4983 - val_accuracy: 0.8804 - lr: 5.0000e-04\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4470 - accuracy: 0.8955 - val_loss: 0.5195 - val_accuracy: 0.8744 - lr: 5.0000e-04\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4441 - accuracy: 0.8933 - val_loss: 0.5706 - val_accuracy: 0.8582 - lr: 5.0000e-04\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.4430 - accuracy: 0.8951 - val_loss: 0.4737 - val_accuracy: 0.8860 - lr: 5.0000e-04\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4318 - accuracy: 0.8995 - val_loss: 0.5548 - val_accuracy: 0.8693 - lr: 5.0000e-04\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.4349 - accuracy: 0.8975 - val_loss: 0.5135 - val_accuracy: 0.8801 - lr: 5.0000e-04\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4326 - accuracy: 0.8970 - val_loss: 0.5299 - val_accuracy: 0.8727 - lr: 5.0000e-04\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4287 - accuracy: 0.8989 - val_loss: 0.5280 - val_accuracy: 0.8712 - lr: 5.0000e-04\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4239 - accuracy: 0.9005 - val_loss: 0.4943 - val_accuracy: 0.8820 - lr: 5.0000e-04\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3865 - accuracy: 0.9126 - val_loss: 0.5047 - val_accuracy: 0.8840 - lr: 2.0000e-04\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3683 - accuracy: 0.9183 - val_loss: 0.4738 - val_accuracy: 0.8901 - lr: 2.0000e-04\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3636 - accuracy: 0.9199 - val_loss: 0.4597 - val_accuracy: 0.8914 - lr: 2.0000e-04\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3575 - accuracy: 0.9204 - val_loss: 0.4689 - val_accuracy: 0.8907 - lr: 2.0000e-04\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.3467 - accuracy: 0.9244 - val_loss: 0.4460 - val_accuracy: 0.8984 - lr: 2.0000e-04\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3455 - accuracy: 0.9248 - val_loss: 0.4421 - val_accuracy: 0.8986 - lr: 2.0000e-04\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.3422 - accuracy: 0.9249 - val_loss: 0.4463 - val_accuracy: 0.8949 - lr: 2.0000e-04\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.3397 - accuracy: 0.9254 - val_loss: 0.4653 - val_accuracy: 0.8922 - lr: 2.0000e-04\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3313 - accuracy: 0.9281 - val_loss: 0.4476 - val_accuracy: 0.8986 - lr: 2.0000e-04\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3294 - accuracy: 0.9271 - val_loss: 0.4588 - val_accuracy: 0.8948 - lr: 2.0000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3283 - accuracy: 0.9274 - val_loss: 0.4485 - val_accuracy: 0.8977 - lr: 2.0000e-04\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.3271 - accuracy: 0.9265 - val_loss: 0.4473 - val_accuracy: 0.8996 - lr: 2.0000e-04\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3225 - accuracy: 0.9283 - val_loss: 0.4558 - val_accuracy: 0.8960 - lr: 2.0000e-04\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3143 - accuracy: 0.9329 - val_loss: 0.4749 - val_accuracy: 0.8878 - lr: 2.0000e-04\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3138 - accuracy: 0.9311 - val_loss: 0.4755 - val_accuracy: 0.8901 - lr: 2.0000e-04\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3132 - accuracy: 0.9313 - val_loss: 0.4393 - val_accuracy: 0.8988 - lr: 2.0000e-04\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3083 - accuracy: 0.9316 - val_loss: 0.4453 - val_accuracy: 0.8977 - lr: 2.0000e-04\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3058 - accuracy: 0.9338 - val_loss: 0.4409 - val_accuracy: 0.9013 - lr: 2.0000e-04\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3061 - accuracy: 0.9337 - val_loss: 0.4368 - val_accuracy: 0.9003 - lr: 2.0000e-04\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.3051 - accuracy: 0.9332 - val_loss: 0.4726 - val_accuracy: 0.8889 - lr: 2.0000e-04\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2896 - accuracy: 0.9367 - val_loss: 0.4326 - val_accuracy: 0.9039 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2811 - accuracy: 0.9394 - val_loss: 0.4420 - val_accuracy: 0.9022 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2823 - accuracy: 0.9399 - val_loss: 0.4334 - val_accuracy: 0.9039 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2789 - accuracy: 0.9420 - val_loss: 0.4283 - val_accuracy: 0.9048 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.2747 - accuracy: 0.9418 - val_loss: 0.4403 - val_accuracy: 0.9020 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2758 - accuracy: 0.9414 - val_loss: 0.4434 - val_accuracy: 0.9010 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2730 - accuracy: 0.9424 - val_loss: 0.4409 - val_accuracy: 0.9018 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2667 - accuracy: 0.9443 - val_loss: 0.4394 - val_accuracy: 0.9028 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2667 - accuracy: 0.9445 - val_loss: 0.4248 - val_accuracy: 0.9055 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2640 - accuracy: 0.9447 - val_loss: 0.4288 - val_accuracy: 0.9041 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 33s 42ms/step - loss: 0.2638 - accuracy: 0.9444 - val_loss: 0.4384 - val_accuracy: 0.9009 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.2636 - accuracy: 0.9449 - val_loss: 0.4171 - val_accuracy: 0.9076 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2621 - accuracy: 0.9461 - val_loss: 0.4365 - val_accuracy: 0.9025 - lr: 1.0000e-04\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2602 - accuracy: 0.9456 - val_loss: 0.4204 - val_accuracy: 0.9071 - lr: 1.0000e-04\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.2592 - accuracy: 0.9453 - val_loss: 0.4406 - val_accuracy: 0.8999 - lr: 1.0000e-04\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2543 - accuracy: 0.9478 - val_loss: 0.4297 - val_accuracy: 0.9042 - lr: 1.0000e-04\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2547 - accuracy: 0.9461 - val_loss: 0.4432 - val_accuracy: 0.9013 - lr: 1.0000e-04\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2539 - accuracy: 0.9474 - val_loss: 0.4461 - val_accuracy: 0.9030 - lr: 1.0000e-04\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2486 - accuracy: 0.9494 - val_loss: 0.4339 - val_accuracy: 0.9035 - lr: 1.0000e-04\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2510 - accuracy: 0.9479 - val_loss: 0.4361 - val_accuracy: 0.9036 - lr: 1.0000e-04\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2459 - accuracy: 0.9491 - val_loss: 0.4254 - val_accuracy: 0.9055 - lr: 5.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2394 - accuracy: 0.9521 - val_loss: 0.4266 - val_accuracy: 0.9057 - lr: 5.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2405 - accuracy: 0.9514 - val_loss: 0.4282 - val_accuracy: 0.9063 - lr: 5.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2397 - accuracy: 0.9517 - val_loss: 0.4319 - val_accuracy: 0.9042 - lr: 5.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2342 - accuracy: 0.9533 - val_loss: 0.4261 - val_accuracy: 0.9054 - lr: 5.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.2337 - accuracy: 0.9528 - val_loss: 0.4262 - val_accuracy: 0.9065 - lr: 5.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2337 - accuracy: 0.9533 - val_loss: 0.4278 - val_accuracy: 0.9062 - lr: 5.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2325 - accuracy: 0.9522 - val_loss: 0.4313 - val_accuracy: 0.9055 - lr: 5.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2380 - accuracy: 0.9514 - val_loss: 0.4285 - val_accuracy: 0.9068 - lr: 5.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2322 - accuracy: 0.9535 - val_loss: 0.4274 - val_accuracy: 0.9042 - lr: 5.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2294 - accuracy: 0.9546 - val_loss: 0.4240 - val_accuracy: 0.9075 - lr: 5.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2310 - accuracy: 0.9544 - val_loss: 0.4248 - val_accuracy: 0.9074 - lr: 5.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2280 - accuracy: 0.9538 - val_loss: 0.4275 - val_accuracy: 0.9048 - lr: 5.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2310 - accuracy: 0.9536 - val_loss: 0.4237 - val_accuracy: 0.9066 - lr: 5.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2296 - accuracy: 0.9539 - val_loss: 0.4204 - val_accuracy: 0.9074 - lr: 5.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2232 - accuracy: 0.9555 - val_loss: 0.4325 - val_accuracy: 0.9063 - lr: 5.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2298 - accuracy: 0.9545 - val_loss: 0.4170 - val_accuracy: 0.9081 - lr: 5.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2281 - accuracy: 0.9538 - val_loss: 0.4305 - val_accuracy: 0.9055 - lr: 5.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2235 - accuracy: 0.9552 - val_loss: 0.4314 - val_accuracy: 0.9056 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x315cb5640>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# increasing dropout after convolutional layers (0.1 to 0.4) and no dropout in fully connected layers\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.datasets import cifar10\n",
    "from keras import regularizers\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(\"........\")\n",
    "print(tf.test.gpu_device_name())\n",
    "print(\"........\")\n",
    "def lr_schedule(epoch):\n",
    "    lrate = 0.001\n",
    "    if epoch > 80:\n",
    "        lrate = 0.00005\n",
    "    elif epoch > 60:\n",
    "        lrate = 0.0001\n",
    "    elif epoch > 40:\n",
    "        lrate = 0.0002\n",
    "    elif epoch > 20:\n",
    "        lrate = 0.0005        \n",
    "    return lrate\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = x_train / 255, x_test / 255\n",
    "\n",
    "num_classes = 10\n",
    "y_train = np_utils.to_categorical(y_train,num_classes)\n",
    "y_test = np_utils.to_categorical(y_test,num_classes)\n",
    "\n",
    "\n",
    "weight_decay = 1e-4\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay), input_shape=x_train.shape[1:]))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.1))\n",
    "\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.2))\n",
    "\n",
    "\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(layers.Dropout(0.3))\n",
    "\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.4))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#data augmentation\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=15,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1,\n",
    "    horizontal_flip=True,\n",
    "    )\n",
    "datagen.fit(x_train)\n",
    "\n",
    "#training\n",
    "batch_size = 64\n",
    "\n",
    "opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ee976b92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........\n",
      "/device:GPU:0\n",
      "........\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 22:01:50.888158: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2021-12-07 22:01:50.888184: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_60 (Conv2D)          (None, 32, 32, 32)        896       \n",
      "                                                                 \n",
      " batch_normalization_68 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_61 (Conv2D)          (None, 32, 32, 32)        9248      \n",
      "                                                                 \n",
      " batch_normalization_69 (Bat  (None, 32, 32, 32)       128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_24 (MaxPoolin  (None, 16, 16, 32)       0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " conv2d_62 (Conv2D)          (None, 16, 16, 64)        18496     \n",
      "                                                                 \n",
      " batch_normalization_70 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_63 (Conv2D)          (None, 16, 16, 64)        36928     \n",
      "                                                                 \n",
      " batch_normalization_71 (Bat  (None, 16, 16, 64)       256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_25 (MaxPoolin  (None, 8, 8, 64)         0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " conv2d_64 (Conv2D)          (None, 8, 8, 128)         73856     \n",
      "                                                                 \n",
      " batch_normalization_72 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_65 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_73 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_66 (Conv2D)          (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_74 (Bat  (None, 8, 8, 128)        512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_26 (MaxPoolin  (None, 4, 4, 128)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " conv2d_67 (Conv2D)          (None, 4, 4, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_75 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_68 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_76 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv2d_69 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_77 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " max_pooling2d_27 (MaxPoolin  (None, 2, 2, 256)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " flatten_6 (Flatten)         (None, 1024)              0         \n",
      "                                                                 \n",
      " dropout_12 (Dropout)        (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_18 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_78 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_13 (Dropout)        (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_19 (Dense)            (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_79 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_20 (Dense)            (None, 10)                10250     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,032,938\n",
      "Trainable params: 4,026,154\n",
      "Non-trainable params: 6,784\n",
      "_________________________________________________________________\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 22:01:52.154336: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "780/781 [============================>.] - ETA: 0s - loss: 1.9336 - accuracy: 0.3796"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 22:02:23.566280: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 36s 42ms/step - loss: 1.9336 - accuracy: 0.3796 - val_loss: 1.9202 - val_accuracy: 0.4632 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 1.3423 - accuracy: 0.5631 - val_loss: 1.1862 - val_accuracy: 0.6191 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 1.0903 - accuracy: 0.6557 - val_loss: 1.4038 - val_accuracy: 0.5671 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.9588 - accuracy: 0.7013 - val_loss: 1.8892 - val_accuracy: 0.5499 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8740 - accuracy: 0.7370 - val_loss: 0.8670 - val_accuracy: 0.7429 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.8158 - accuracy: 0.7615 - val_loss: 0.9448 - val_accuracy: 0.7334 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7750 - accuracy: 0.7789 - val_loss: 0.7491 - val_accuracy: 0.7886 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7380 - accuracy: 0.7930 - val_loss: 0.8424 - val_accuracy: 0.7647 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7099 - accuracy: 0.8029 - val_loss: 0.8536 - val_accuracy: 0.7674 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.6919 - accuracy: 0.8120 - val_loss: 0.7263 - val_accuracy: 0.7979 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6695 - accuracy: 0.8195 - val_loss: 0.7740 - val_accuracy: 0.7919 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6513 - accuracy: 0.8283 - val_loss: 0.6728 - val_accuracy: 0.8239 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.6369 - accuracy: 0.8348 - val_loss: 0.7346 - val_accuracy: 0.8054 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6259 - accuracy: 0.8397 - val_loss: 0.6552 - val_accuracy: 0.8325 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6103 - accuracy: 0.8454 - val_loss: 0.7454 - val_accuracy: 0.8050 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.6007 - accuracy: 0.8498 - val_loss: 0.6937 - val_accuracy: 0.8251 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5962 - accuracy: 0.8542 - val_loss: 0.6215 - val_accuracy: 0.8441 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5830 - accuracy: 0.8581 - val_loss: 0.7573 - val_accuracy: 0.8043 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5789 - accuracy: 0.8615 - val_loss: 0.7597 - val_accuracy: 0.7978 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5747 - accuracy: 0.8631 - val_loss: 0.6816 - val_accuracy: 0.8326 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.5612 - accuracy: 0.8680 - val_loss: 0.7348 - val_accuracy: 0.8160 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4851 - accuracy: 0.8915 - val_loss: 0.5660 - val_accuracy: 0.8630 - lr: 5.0000e-04\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4568 - accuracy: 0.8993 - val_loss: 0.5785 - val_accuracy: 0.8629 - lr: 5.0000e-04\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4427 - accuracy: 0.9040 - val_loss: 0.5347 - val_accuracy: 0.8749 - lr: 5.0000e-04\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4309 - accuracy: 0.9065 - val_loss: 0.5724 - val_accuracy: 0.8631 - lr: 5.0000e-04\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4197 - accuracy: 0.9084 - val_loss: 0.5406 - val_accuracy: 0.8737 - lr: 5.0000e-04\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4166 - accuracy: 0.9085 - val_loss: 0.5761 - val_accuracy: 0.8669 - lr: 5.0000e-04\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4064 - accuracy: 0.9112 - val_loss: 0.4907 - val_accuracy: 0.8850 - lr: 5.0000e-04\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4032 - accuracy: 0.9121 - val_loss: 0.5531 - val_accuracy: 0.8664 - lr: 5.0000e-04\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3945 - accuracy: 0.9152 - val_loss: 0.5292 - val_accuracy: 0.8754 - lr: 5.0000e-04\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3882 - accuracy: 0.9156 - val_loss: 0.5915 - val_accuracy: 0.8584 - lr: 5.0000e-04\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3832 - accuracy: 0.9161 - val_loss: 0.5226 - val_accuracy: 0.8779 - lr: 5.0000e-04\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3799 - accuracy: 0.9174 - val_loss: 0.4990 - val_accuracy: 0.8845 - lr: 5.0000e-04\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3758 - accuracy: 0.9192 - val_loss: 0.5106 - val_accuracy: 0.8796 - lr: 5.0000e-04\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3695 - accuracy: 0.9209 - val_loss: 0.5236 - val_accuracy: 0.8772 - lr: 5.0000e-04\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3653 - accuracy: 0.9225 - val_loss: 0.5292 - val_accuracy: 0.8779 - lr: 5.0000e-04\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.3638 - accuracy: 0.9237 - val_loss: 0.5204 - val_accuracy: 0.8796 - lr: 5.0000e-04\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3589 - accuracy: 0.9256 - val_loss: 0.5538 - val_accuracy: 0.8711 - lr: 5.0000e-04\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3521 - accuracy: 0.9262 - val_loss: 0.5162 - val_accuracy: 0.8827 - lr: 5.0000e-04\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.3561 - accuracy: 0.9248 - val_loss: 0.5548 - val_accuracy: 0.8713 - lr: 5.0000e-04\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3489 - accuracy: 0.9269 - val_loss: 0.5160 - val_accuracy: 0.8752 - lr: 5.0000e-04\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.3049 - accuracy: 0.9419 - val_loss: 0.5094 - val_accuracy: 0.8903 - lr: 2.0000e-04\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2887 - accuracy: 0.9466 - val_loss: 0.4651 - val_accuracy: 0.8992 - lr: 2.0000e-04\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2789 - accuracy: 0.9502 - val_loss: 0.4767 - val_accuracy: 0.8967 - lr: 2.0000e-04\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2763 - accuracy: 0.9499 - val_loss: 0.4772 - val_accuracy: 0.8964 - lr: 2.0000e-04\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2717 - accuracy: 0.9513 - val_loss: 0.4656 - val_accuracy: 0.8972 - lr: 2.0000e-04\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2645 - accuracy: 0.9540 - val_loss: 0.4854 - val_accuracy: 0.8932 - lr: 2.0000e-04\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2625 - accuracy: 0.9529 - val_loss: 0.4725 - val_accuracy: 0.8975 - lr: 2.0000e-04\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2580 - accuracy: 0.9543 - val_loss: 0.4708 - val_accuracy: 0.8969 - lr: 2.0000e-04\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2540 - accuracy: 0.9539 - val_loss: 0.4953 - val_accuracy: 0.8935 - lr: 2.0000e-04\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2520 - accuracy: 0.9574 - val_loss: 0.4787 - val_accuracy: 0.8958 - lr: 2.0000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2489 - accuracy: 0.9570 - val_loss: 0.4769 - val_accuracy: 0.8948 - lr: 2.0000e-04\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2467 - accuracy: 0.9568 - val_loss: 0.4546 - val_accuracy: 0.9021 - lr: 2.0000e-04\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2424 - accuracy: 0.9574 - val_loss: 0.4658 - val_accuracy: 0.9021 - lr: 2.0000e-04\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2420 - accuracy: 0.9587 - val_loss: 0.4732 - val_accuracy: 0.8973 - lr: 2.0000e-04\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2378 - accuracy: 0.9585 - val_loss: 0.4858 - val_accuracy: 0.8949 - lr: 2.0000e-04\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2308 - accuracy: 0.9610 - val_loss: 0.4786 - val_accuracy: 0.8989 - lr: 2.0000e-04\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.2334 - accuracy: 0.9593 - val_loss: 0.4636 - val_accuracy: 0.8998 - lr: 2.0000e-04\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2291 - accuracy: 0.9612 - val_loss: 0.4910 - val_accuracy: 0.8963 - lr: 2.0000e-04\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2258 - accuracy: 0.9622 - val_loss: 0.4737 - val_accuracy: 0.9005 - lr: 2.0000e-04\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.2276 - accuracy: 0.9611 - val_loss: 0.4949 - val_accuracy: 0.8947 - lr: 2.0000e-04\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2093 - accuracy: 0.9665 - val_loss: 0.4570 - val_accuracy: 0.9067 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.2003 - accuracy: 0.9689 - val_loss: 0.4733 - val_accuracy: 0.9023 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.2024 - accuracy: 0.9689 - val_loss: 0.4730 - val_accuracy: 0.9033 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1981 - accuracy: 0.9700 - val_loss: 0.4781 - val_accuracy: 0.9015 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1950 - accuracy: 0.9709 - val_loss: 0.4653 - val_accuracy: 0.9058 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1922 - accuracy: 0.9712 - val_loss: 0.4873 - val_accuracy: 0.9028 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1913 - accuracy: 0.9718 - val_loss: 0.4777 - val_accuracy: 0.9036 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1894 - accuracy: 0.9719 - val_loss: 0.4693 - val_accuracy: 0.9054 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1895 - accuracy: 0.9718 - val_loss: 0.4722 - val_accuracy: 0.9039 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1863 - accuracy: 0.9723 - val_loss: 0.4779 - val_accuracy: 0.9048 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1862 - accuracy: 0.9734 - val_loss: 0.4829 - val_accuracy: 0.9018 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1825 - accuracy: 0.9740 - val_loss: 0.4726 - val_accuracy: 0.9046 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1831 - accuracy: 0.9731 - val_loss: 0.4724 - val_accuracy: 0.9063 - lr: 1.0000e-04\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1819 - accuracy: 0.9737 - val_loss: 0.4695 - val_accuracy: 0.9064 - lr: 1.0000e-04\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1798 - accuracy: 0.9739 - val_loss: 0.4845 - val_accuracy: 0.9018 - lr: 1.0000e-04\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1781 - accuracy: 0.9750 - val_loss: 0.4743 - val_accuracy: 0.9073 - lr: 1.0000e-04\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1765 - accuracy: 0.9758 - val_loss: 0.4717 - val_accuracy: 0.9069 - lr: 1.0000e-04\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1786 - accuracy: 0.9747 - val_loss: 0.4782 - val_accuracy: 0.9051 - lr: 1.0000e-04\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1741 - accuracy: 0.9755 - val_loss: 0.4682 - val_accuracy: 0.9037 - lr: 1.0000e-04\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1731 - accuracy: 0.9756 - val_loss: 0.4788 - val_accuracy: 0.9049 - lr: 1.0000e-04\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1697 - accuracy: 0.9772 - val_loss: 0.4661 - val_accuracy: 0.9086 - lr: 5.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1615 - accuracy: 0.9798 - val_loss: 0.4815 - val_accuracy: 0.9052 - lr: 5.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1629 - accuracy: 0.9783 - val_loss: 0.4757 - val_accuracy: 0.9080 - lr: 5.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1605 - accuracy: 0.9790 - val_loss: 0.4712 - val_accuracy: 0.9074 - lr: 5.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1624 - accuracy: 0.9783 - val_loss: 0.4816 - val_accuracy: 0.9076 - lr: 5.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1612 - accuracy: 0.9790 - val_loss: 0.4678 - val_accuracy: 0.9088 - lr: 5.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1583 - accuracy: 0.9805 - val_loss: 0.4705 - val_accuracy: 0.9095 - lr: 5.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1601 - accuracy: 0.9796 - val_loss: 0.4767 - val_accuracy: 0.9076 - lr: 5.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1566 - accuracy: 0.9806 - val_loss: 0.4776 - val_accuracy: 0.9077 - lr: 5.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1570 - accuracy: 0.9797 - val_loss: 0.4788 - val_accuracy: 0.9070 - lr: 5.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1542 - accuracy: 0.9810 - val_loss: 0.4838 - val_accuracy: 0.9071 - lr: 5.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 29s 38ms/step - loss: 0.1533 - accuracy: 0.9807 - val_loss: 0.4788 - val_accuracy: 0.9074 - lr: 5.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1562 - accuracy: 0.9808 - val_loss: 0.4797 - val_accuracy: 0.9098 - lr: 5.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1526 - accuracy: 0.9809 - val_loss: 0.4851 - val_accuracy: 0.9074 - lr: 5.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1530 - accuracy: 0.9813 - val_loss: 0.4700 - val_accuracy: 0.9108 - lr: 5.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1502 - accuracy: 0.9818 - val_loss: 0.4927 - val_accuracy: 0.9062 - lr: 5.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1496 - accuracy: 0.9822 - val_loss: 0.4849 - val_accuracy: 0.9092 - lr: 5.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1493 - accuracy: 0.9821 - val_loss: 0.4825 - val_accuracy: 0.9068 - lr: 5.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1492 - accuracy: 0.9818 - val_loss: 0.4957 - val_accuracy: 0.9051 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x3159964c0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropout applied only on fully connected layers \n",
    "# this is what the original paper on dropout proposed:\n",
    "#      https://arxiv.org/pdf/1207.0580.pdf\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.datasets import cifar10\n",
    "from keras import regularizers\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(\"........\")\n",
    "print(tf.test.gpu_device_name())\n",
    "print(\"........\")\n",
    "def lr_schedule(epoch):\n",
    "    lrate = 0.001\n",
    "    if epoch > 80:\n",
    "        lrate = 0.00005\n",
    "    elif epoch > 60:\n",
    "        lrate = 0.0001\n",
    "    elif epoch > 40:\n",
    "        lrate = 0.0002\n",
    "    elif epoch > 20:\n",
    "        lrate = 0.0005        \n",
    "    return lrate\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = x_train / 255, x_test / 255\n",
    "\n",
    "#z-score\n",
    "\n",
    "\n",
    "num_classes = 10\n",
    "y_train = np_utils.to_categorical(y_train,num_classes)\n",
    "y_test = np_utils.to_categorical(y_test,num_classes)\n",
    "\n",
    "weight_decay = 1e-4\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay), input_shape=x_train.shape[1:]))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(layers.BatchNormalization())\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#data augmentation\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=15,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1,\n",
    "    horizontal_flip=True,\n",
    "    )\n",
    "datagen.fit(x_train)\n",
    "\n",
    "#training\n",
    "batch_size = 64\n",
    "\n",
    "opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b39ccc5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 22:54:12.652394: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "780/781 [============================>.] - ETA: 0s - loss: 0.1457 - accuracy: 0.9829"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-07 22:54:44.796663: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 36s 41ms/step - loss: 0.1457 - accuracy: 0.9830 - val_loss: 0.4826 - val_accuracy: 0.9075 - lr: 2.0000e-05\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1449 - accuracy: 0.9834 - val_loss: 0.4825 - val_accuracy: 0.9089 - lr: 2.0000e-05\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1432 - accuracy: 0.9839 - val_loss: 0.4880 - val_accuracy: 0.9089 - lr: 2.0000e-05\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1466 - accuracy: 0.9829 - val_loss: 0.4916 - val_accuracy: 0.9074 - lr: 2.0000e-05\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1423 - accuracy: 0.9839 - val_loss: 0.4821 - val_accuracy: 0.9093 - lr: 2.0000e-05\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1450 - accuracy: 0.9828 - val_loss: 0.4798 - val_accuracy: 0.9109 - lr: 2.0000e-05\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1419 - accuracy: 0.9841 - val_loss: 0.4838 - val_accuracy: 0.9090 - lr: 2.0000e-05\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1434 - accuracy: 0.9830 - val_loss: 0.4862 - val_accuracy: 0.9091 - lr: 2.0000e-05\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1421 - accuracy: 0.9836 - val_loss: 0.4826 - val_accuracy: 0.9091 - lr: 2.0000e-05\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1419 - accuracy: 0.9839 - val_loss: 0.4781 - val_accuracy: 0.9107 - lr: 2.0000e-05\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1409 - accuracy: 0.9841 - val_loss: 0.4826 - val_accuracy: 0.9102 - lr: 2.0000e-05\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1391 - accuracy: 0.9846 - val_loss: 0.4860 - val_accuracy: 0.9103 - lr: 2.0000e-05\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1388 - accuracy: 0.9847 - val_loss: 0.4901 - val_accuracy: 0.9097 - lr: 2.0000e-05\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1410 - accuracy: 0.9840 - val_loss: 0.4888 - val_accuracy: 0.9087 - lr: 2.0000e-05\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1395 - accuracy: 0.9840 - val_loss: 0.4919 - val_accuracy: 0.9086 - lr: 2.0000e-05\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1399 - accuracy: 0.9845 - val_loss: 0.4829 - val_accuracy: 0.9102 - lr: 2.0000e-05\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1392 - accuracy: 0.9845 - val_loss: 0.4910 - val_accuracy: 0.9076 - lr: 2.0000e-05\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1390 - accuracy: 0.9845 - val_loss: 0.4982 - val_accuracy: 0.9077 - lr: 2.0000e-05\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1361 - accuracy: 0.9854 - val_loss: 0.5008 - val_accuracy: 0.9094 - lr: 2.0000e-05\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1395 - accuracy: 0.9843 - val_loss: 0.4984 - val_accuracy: 0.9080 - lr: 2.0000e-05\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1390 - accuracy: 0.9840 - val_loss: 0.4988 - val_accuracy: 0.9079 - lr: 2.0000e-05\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1361 - accuracy: 0.9853 - val_loss: 0.4996 - val_accuracy: 0.9079 - lr: 2.0000e-05\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1345 - accuracy: 0.9857 - val_loss: 0.4921 - val_accuracy: 0.9095 - lr: 2.0000e-05\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1359 - accuracy: 0.9857 - val_loss: 0.4964 - val_accuracy: 0.9087 - lr: 2.0000e-05\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1358 - accuracy: 0.9851 - val_loss: 0.5002 - val_accuracy: 0.9089 - lr: 2.0000e-05\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1321 - accuracy: 0.9863 - val_loss: 0.4963 - val_accuracy: 0.9091 - lr: 1.0000e-05\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1352 - accuracy: 0.9855 - val_loss: 0.4977 - val_accuracy: 0.9094 - lr: 1.0000e-05\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1365 - accuracy: 0.9851 - val_loss: 0.5003 - val_accuracy: 0.9087 - lr: 1.0000e-05\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1337 - accuracy: 0.9859 - val_loss: 0.4980 - val_accuracy: 0.9084 - lr: 1.0000e-05\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1349 - accuracy: 0.9862 - val_loss: 0.4919 - val_accuracy: 0.9105 - lr: 1.0000e-05\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1348 - accuracy: 0.9857 - val_loss: 0.4964 - val_accuracy: 0.9087 - lr: 1.0000e-05\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1337 - accuracy: 0.9860 - val_loss: 0.4987 - val_accuracy: 0.9084 - lr: 1.0000e-05\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1352 - accuracy: 0.9859 - val_loss: 0.4994 - val_accuracy: 0.9087 - lr: 1.0000e-05\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1340 - accuracy: 0.9858 - val_loss: 0.4973 - val_accuracy: 0.9080 - lr: 1.0000e-05\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1343 - accuracy: 0.9856 - val_loss: 0.4996 - val_accuracy: 0.9084 - lr: 1.0000e-05\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1324 - accuracy: 0.9862 - val_loss: 0.4970 - val_accuracy: 0.9079 - lr: 1.0000e-05\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1321 - accuracy: 0.9866 - val_loss: 0.4952 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1330 - accuracy: 0.9857 - val_loss: 0.4957 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1341 - accuracy: 0.9862 - val_loss: 0.4980 - val_accuracy: 0.9086 - lr: 1.0000e-05\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1323 - accuracy: 0.9864 - val_loss: 0.4969 - val_accuracy: 0.9091 - lr: 1.0000e-05\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.1324 - accuracy: 0.9860 - val_loss: 0.4953 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1318 - accuracy: 0.9861 - val_loss: 0.5021 - val_accuracy: 0.9084 - lr: 1.0000e-05\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1324 - accuracy: 0.9857 - val_loss: 0.5010 - val_accuracy: 0.9090 - lr: 1.0000e-05\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1314 - accuracy: 0.9872 - val_loss: 0.4999 - val_accuracy: 0.9096 - lr: 1.0000e-05\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1325 - accuracy: 0.9862 - val_loss: 0.5002 - val_accuracy: 0.9095 - lr: 1.0000e-05\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1308 - accuracy: 0.9865 - val_loss: 0.4970 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1314 - accuracy: 0.9866 - val_loss: 0.4971 - val_accuracy: 0.9103 - lr: 1.0000e-05\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1319 - accuracy: 0.9867 - val_loss: 0.4942 - val_accuracy: 0.9110 - lr: 1.0000e-05\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1295 - accuracy: 0.9869 - val_loss: 0.4931 - val_accuracy: 0.9116 - lr: 1.0000e-05\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1324 - accuracy: 0.9860 - val_loss: 0.5006 - val_accuracy: 0.9089 - lr: 1.0000e-05\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1312 - accuracy: 0.9860 - val_loss: 0.4987 - val_accuracy: 0.9089 - lr: 1.0000e-05\n",
      "Epoch 52/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1309 - accuracy: 0.9868 - val_loss: 0.4986 - val_accuracy: 0.9107 - lr: 1.0000e-05\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1317 - accuracy: 0.9864 - val_loss: 0.5008 - val_accuracy: 0.9090 - lr: 1.0000e-05\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1303 - accuracy: 0.9865 - val_loss: 0.4967 - val_accuracy: 0.9098 - lr: 1.0000e-05\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1312 - accuracy: 0.9866 - val_loss: 0.5031 - val_accuracy: 0.9083 - lr: 1.0000e-05\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1310 - accuracy: 0.9863 - val_loss: 0.5020 - val_accuracy: 0.9091 - lr: 1.0000e-05\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1299 - accuracy: 0.9865 - val_loss: 0.5018 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1291 - accuracy: 0.9875 - val_loss: 0.5031 - val_accuracy: 0.9083 - lr: 1.0000e-05\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1304 - accuracy: 0.9864 - val_loss: 0.5025 - val_accuracy: 0.9080 - lr: 1.0000e-05\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1282 - accuracy: 0.9864 - val_loss: 0.5008 - val_accuracy: 0.9096 - lr: 1.0000e-05\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1321 - accuracy: 0.9862 - val_loss: 0.4978 - val_accuracy: 0.9108 - lr: 1.0000e-05\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1269 - accuracy: 0.9875 - val_loss: 0.5042 - val_accuracy: 0.9091 - lr: 1.0000e-05\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1282 - accuracy: 0.9877 - val_loss: 0.5079 - val_accuracy: 0.9088 - lr: 1.0000e-05\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1281 - accuracy: 0.9870 - val_loss: 0.5011 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1293 - accuracy: 0.9872 - val_loss: 0.5061 - val_accuracy: 0.9095 - lr: 1.0000e-05\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1274 - accuracy: 0.9872 - val_loss: 0.5033 - val_accuracy: 0.9098 - lr: 1.0000e-05\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1285 - accuracy: 0.9875 - val_loss: 0.5050 - val_accuracy: 0.9089 - lr: 1.0000e-05\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1276 - accuracy: 0.9870 - val_loss: 0.5062 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.1289 - accuracy: 0.9869 - val_loss: 0.5084 - val_accuracy: 0.9098 - lr: 1.0000e-05\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1253 - accuracy: 0.9882 - val_loss: 0.5052 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1297 - accuracy: 0.9868 - val_loss: 0.5052 - val_accuracy: 0.9096 - lr: 1.0000e-05\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1281 - accuracy: 0.9879 - val_loss: 0.4988 - val_accuracy: 0.9101 - lr: 1.0000e-05\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1288 - accuracy: 0.9873 - val_loss: 0.5051 - val_accuracy: 0.9081 - lr: 1.0000e-05\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1276 - accuracy: 0.9875 - val_loss: 0.5054 - val_accuracy: 0.9087 - lr: 1.0000e-05\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1295 - accuracy: 0.9869 - val_loss: 0.5037 - val_accuracy: 0.9090 - lr: 1.0000e-05\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1269 - accuracy: 0.9877 - val_loss: 0.5026 - val_accuracy: 0.9096 - lr: 1.0000e-05\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.1271 - accuracy: 0.9871 - val_loss: 0.5100 - val_accuracy: 0.9081 - lr: 1.0000e-05\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1283 - accuracy: 0.9869 - val_loss: 0.5076 - val_accuracy: 0.9098 - lr: 1.0000e-05\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1269 - accuracy: 0.9869 - val_loss: 0.5057 - val_accuracy: 0.9101 - lr: 1.0000e-05\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1280 - accuracy: 0.9869 - val_loss: 0.5074 - val_accuracy: 0.9091 - lr: 1.0000e-05\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1283 - accuracy: 0.9868 - val_loss: 0.5053 - val_accuracy: 0.9101 - lr: 1.0000e-05\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1251 - accuracy: 0.9878 - val_loss: 0.5052 - val_accuracy: 0.9089 - lr: 1.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1247 - accuracy: 0.9878 - val_loss: 0.5070 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1264 - accuracy: 0.9869 - val_loss: 0.5098 - val_accuracy: 0.9087 - lr: 1.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1266 - accuracy: 0.9872 - val_loss: 0.5083 - val_accuracy: 0.9105 - lr: 1.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1274 - accuracy: 0.9868 - val_loss: 0.5078 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1264 - accuracy: 0.9874 - val_loss: 0.5054 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1267 - accuracy: 0.9877 - val_loss: 0.5048 - val_accuracy: 0.9105 - lr: 1.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1258 - accuracy: 0.9875 - val_loss: 0.5087 - val_accuracy: 0.9093 - lr: 1.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1245 - accuracy: 0.9875 - val_loss: 0.5060 - val_accuracy: 0.9100 - lr: 1.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1240 - accuracy: 0.9882 - val_loss: 0.5073 - val_accuracy: 0.9108 - lr: 1.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1269 - accuracy: 0.9869 - val_loss: 0.5085 - val_accuracy: 0.9110 - lr: 1.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1266 - accuracy: 0.9874 - val_loss: 0.5056 - val_accuracy: 0.9115 - lr: 1.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1266 - accuracy: 0.9871 - val_loss: 0.5081 - val_accuracy: 0.9108 - lr: 1.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1250 - accuracy: 0.9872 - val_loss: 0.5098 - val_accuracy: 0.9102 - lr: 1.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.1227 - accuracy: 0.9879 - val_loss: 0.5074 - val_accuracy: 0.9108 - lr: 1.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1231 - accuracy: 0.9884 - val_loss: 0.5061 - val_accuracy: 0.9120 - lr: 1.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1247 - accuracy: 0.9876 - val_loss: 0.5119 - val_accuracy: 0.9099 - lr: 1.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.1259 - accuracy: 0.9877 - val_loss: 0.5138 - val_accuracy: 0.9104 - lr: 1.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.1259 - accuracy: 0.9872 - val_loss: 0.5115 - val_accuracy: 0.9088 - lr: 1.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x3159557c0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# additional 100 epochs of the previous model\n",
    "def lr(epoch):\n",
    "    lrate = 0.00002\n",
    "    if epoch >= 25:\n",
    "        lrate = 0.00001     \n",
    "    return lrate\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "11de8865",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........\n",
      "/device:GPU:0\n",
      "........\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-08 13:50:24.919739: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2021-12-08 13:50:24.919761: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_2 (Conv2D)           (None, 32, 32, 32)        896       \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 32, 32, 32)       128       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 32, 32, 32)        0         \n",
      "                                                                 \n",
      " conv2d_3 (Conv2D)           (None, 32, 32, 32)        9248      \n",
      "                                                                 \n",
      " batch_normalization_3 (Batc  (None, 32, 32, 32)       128       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 32, 32, 32)        0         \n",
      "                                                                 \n",
      " max_pooling2d (MaxPooling2D  (None, 16, 16, 32)       0         \n",
      " )                                                               \n",
      "                                                                 \n",
      " conv2d_4 (Conv2D)           (None, 16, 16, 64)        18496     \n",
      "                                                                 \n",
      " batch_normalization_4 (Batc  (None, 16, 16, 64)       256       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 16, 16, 64)        0         \n",
      "                                                                 \n",
      " conv2d_5 (Conv2D)           (None, 16, 16, 64)        36928     \n",
      "                                                                 \n",
      " batch_normalization_5 (Batc  (None, 16, 16, 64)       256       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_3 (Dropout)         (None, 16, 16, 64)        0         \n",
      "                                                                 \n",
      " max_pooling2d_1 (MaxPooling  (None, 8, 8, 64)         0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_6 (Conv2D)           (None, 8, 8, 128)         73856     \n",
      "                                                                 \n",
      " batch_normalization_6 (Batc  (None, 8, 8, 128)        512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_4 (Dropout)         (None, 8, 8, 128)         0         \n",
      "                                                                 \n",
      " conv2d_7 (Conv2D)           (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_7 (Batc  (None, 8, 8, 128)        512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_5 (Dropout)         (None, 8, 8, 128)         0         \n",
      "                                                                 \n",
      " conv2d_8 (Conv2D)           (None, 8, 8, 128)         147584    \n",
      "                                                                 \n",
      " batch_normalization_8 (Batc  (None, 8, 8, 128)        512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_6 (Dropout)         (None, 8, 8, 128)         0         \n",
      "                                                                 \n",
      " max_pooling2d_2 (MaxPooling  (None, 4, 4, 128)        0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_9 (Conv2D)           (None, 4, 4, 256)         295168    \n",
      "                                                                 \n",
      " batch_normalization_9 (Batc  (None, 4, 4, 256)        1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dropout_7 (Dropout)         (None, 4, 4, 256)         0         \n",
      "                                                                 \n",
      " conv2d_10 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_10 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_8 (Dropout)         (None, 4, 4, 256)         0         \n",
      "                                                                 \n",
      " conv2d_11 (Conv2D)          (None, 4, 4, 256)         590080    \n",
      "                                                                 \n",
      " batch_normalization_11 (Bat  (None, 4, 4, 256)        1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_9 (Dropout)         (None, 4, 4, 256)         0         \n",
      "                                                                 \n",
      " max_pooling2d_3 (MaxPooling  (None, 2, 2, 256)        0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 1024)              0         \n",
      "                                                                 \n",
      " dropout_10 (Dropout)        (None, 1024)              0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_12 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_11 (Dropout)        (None, 1024)              0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1024)              1049600   \n",
      "                                                                 \n",
      " batch_normalization_13 (Bat  (None, 1024)             4096      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 10)                10250     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,032,938\n",
      "Trainable params: 4,026,154\n",
      "Non-trainable params: 6,784\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanrongqi/miniforge3/envs/tf/lib/python3.9/site-packages/keras/optimizer_v2/rmsprop.py:130: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(RMSprop, self).__init__(name, **kwargs)\n",
      "2021-12-08 13:50:25.705341: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-08 13:50:26.429749: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - ETA: 0s - loss: 2.1328 - accuracy: 0.2893"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-08 13:50:56.299195: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "781/781 [==============================] - 34s 41ms/step - loss: 2.1328 - accuracy: 0.2893 - val_loss: 2.7007 - val_accuracy: 0.1977 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 1.6159 - accuracy: 0.4616 - val_loss: 5.4362 - val_accuracy: 0.3508 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 1.4254 - accuracy: 0.5423 - val_loss: 1.2727 - val_accuracy: 0.5913 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 1.2601 - accuracy: 0.6002 - val_loss: 2.7781 - val_accuracy: 0.5448 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 1.1586 - accuracy: 0.6384 - val_loss: 1.3218 - val_accuracy: 0.5755 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 1.0852 - accuracy: 0.6668 - val_loss: 1.0710 - val_accuracy: 0.6793 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 1.0346 - accuracy: 0.6904 - val_loss: 1.5251 - val_accuracy: 0.6163 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.9941 - accuracy: 0.7032 - val_loss: 0.9326 - val_accuracy: 0.7429 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.9610 - accuracy: 0.7201 - val_loss: 1.2299 - val_accuracy: 0.6743 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.9336 - accuracy: 0.7289 - val_loss: 1.2002 - val_accuracy: 0.7180 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.9093 - accuracy: 0.7375 - val_loss: 1.2845 - val_accuracy: 0.6816 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8969 - accuracy: 0.7456 - val_loss: 0.8684 - val_accuracy: 0.7652 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8786 - accuracy: 0.7503 - val_loss: 0.8807 - val_accuracy: 0.7575 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.8676 - accuracy: 0.7563 - val_loss: 1.2234 - val_accuracy: 0.6949 - lr: 0.0010\n",
      "Epoch 15/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8589 - accuracy: 0.7606 - val_loss: 0.9876 - val_accuracy: 0.7245 - lr: 0.0010\n",
      "Epoch 16/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8468 - accuracy: 0.7662 - val_loss: 1.0896 - val_accuracy: 0.7218 - lr: 0.0010\n",
      "Epoch 17/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8317 - accuracy: 0.7717 - val_loss: 0.7632 - val_accuracy: 0.7967 - lr: 0.0010\n",
      "Epoch 18/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8302 - accuracy: 0.7714 - val_loss: 0.9300 - val_accuracy: 0.7455 - lr: 0.0010\n",
      "Epoch 19/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8181 - accuracy: 0.7782 - val_loss: 0.7552 - val_accuracy: 0.7958 - lr: 0.0010\n",
      "Epoch 20/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8102 - accuracy: 0.7821 - val_loss: 0.8180 - val_accuracy: 0.7729 - lr: 0.0010\n",
      "Epoch 21/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.8095 - accuracy: 0.7827 - val_loss: 0.8204 - val_accuracy: 0.7749 - lr: 0.0010\n",
      "Epoch 22/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7478 - accuracy: 0.8020 - val_loss: 0.7301 - val_accuracy: 0.8075 - lr: 5.0000e-04\n",
      "Epoch 23/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.7180 - accuracy: 0.8116 - val_loss: 0.6627 - val_accuracy: 0.8280 - lr: 5.0000e-04\n",
      "Epoch 24/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.7045 - accuracy: 0.8138 - val_loss: 0.7015 - val_accuracy: 0.8108 - lr: 5.0000e-04\n",
      "Epoch 25/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6985 - accuracy: 0.8163 - val_loss: 0.7256 - val_accuracy: 0.8027 - lr: 5.0000e-04\n",
      "Epoch 26/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6898 - accuracy: 0.8166 - val_loss: 0.6971 - val_accuracy: 0.8110 - lr: 5.0000e-04\n",
      "Epoch 27/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6825 - accuracy: 0.8178 - val_loss: 0.6832 - val_accuracy: 0.8154 - lr: 5.0000e-04\n",
      "Epoch 28/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6713 - accuracy: 0.8179 - val_loss: 0.6862 - val_accuracy: 0.8099 - lr: 5.0000e-04\n",
      "Epoch 29/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6658 - accuracy: 0.8216 - val_loss: 0.6661 - val_accuracy: 0.8227 - lr: 5.0000e-04\n",
      "Epoch 30/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6662 - accuracy: 0.8215 - val_loss: 0.6483 - val_accuracy: 0.8257 - lr: 5.0000e-04\n",
      "Epoch 31/100\n",
      "781/781 [==============================] - 33s 42ms/step - loss: 0.6612 - accuracy: 0.8223 - val_loss: 0.6703 - val_accuracy: 0.8195 - lr: 5.0000e-04\n",
      "Epoch 32/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6507 - accuracy: 0.8255 - val_loss: 0.6779 - val_accuracy: 0.8162 - lr: 5.0000e-04\n",
      "Epoch 33/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6529 - accuracy: 0.8251 - val_loss: 0.6132 - val_accuracy: 0.8338 - lr: 5.0000e-04\n",
      "Epoch 34/100\n",
      "781/781 [==============================] - 957s 1s/step - loss: 0.6487 - accuracy: 0.8274 - val_loss: 0.6734 - val_accuracy: 0.8167 - lr: 5.0000e-04\n",
      "Epoch 35/100\n",
      "781/781 [==============================] - 1258s 2s/step - loss: 0.6436 - accuracy: 0.8276 - val_loss: 0.6004 - val_accuracy: 0.8392 - lr: 5.0000e-04\n",
      "Epoch 36/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6358 - accuracy: 0.8311 - val_loss: 0.7070 - val_accuracy: 0.8012 - lr: 5.0000e-04\n",
      "Epoch 37/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6334 - accuracy: 0.8304 - val_loss: 0.7043 - val_accuracy: 0.8071 - lr: 5.0000e-04\n",
      "Epoch 38/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.6289 - accuracy: 0.8337 - val_loss: 0.7348 - val_accuracy: 0.7929 - lr: 5.0000e-04\n",
      "Epoch 39/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.6258 - accuracy: 0.8331 - val_loss: 0.6548 - val_accuracy: 0.8154 - lr: 5.0000e-04\n",
      "Epoch 40/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.6267 - accuracy: 0.8327 - val_loss: 0.6492 - val_accuracy: 0.8192 - lr: 5.0000e-04\n",
      "Epoch 41/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.6241 - accuracy: 0.8353 - val_loss: 0.6952 - val_accuracy: 0.8048 - lr: 5.0000e-04\n",
      "Epoch 42/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5897 - accuracy: 0.8446 - val_loss: 0.5804 - val_accuracy: 0.8457 - lr: 2.0000e-04\n",
      "Epoch 43/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5775 - accuracy: 0.8478 - val_loss: 0.6390 - val_accuracy: 0.8232 - lr: 2.0000e-04\n",
      "Epoch 44/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5656 - accuracy: 0.8521 - val_loss: 0.5956 - val_accuracy: 0.8369 - lr: 2.0000e-04\n",
      "Epoch 45/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5587 - accuracy: 0.8521 - val_loss: 0.5833 - val_accuracy: 0.8432 - lr: 2.0000e-04\n",
      "Epoch 46/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5509 - accuracy: 0.8552 - val_loss: 0.5924 - val_accuracy: 0.8381 - lr: 2.0000e-04\n",
      "Epoch 47/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5545 - accuracy: 0.8536 - val_loss: 0.6254 - val_accuracy: 0.8282 - lr: 2.0000e-04\n",
      "Epoch 48/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5475 - accuracy: 0.8546 - val_loss: 0.5538 - val_accuracy: 0.8529 - lr: 2.0000e-04\n",
      "Epoch 49/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5539 - accuracy: 0.8543 - val_loss: 0.6041 - val_accuracy: 0.8340 - lr: 2.0000e-04\n",
      "Epoch 50/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.5379 - accuracy: 0.8590 - val_loss: 0.5762 - val_accuracy: 0.8440 - lr: 2.0000e-04\n",
      "Epoch 51/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5398 - accuracy: 0.8555 - val_loss: 0.5912 - val_accuracy: 0.8366 - lr: 2.0000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5375 - accuracy: 0.8570 - val_loss: 0.5664 - val_accuracy: 0.8467 - lr: 2.0000e-04\n",
      "Epoch 53/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.5362 - accuracy: 0.8567 - val_loss: 0.5696 - val_accuracy: 0.8444 - lr: 2.0000e-04\n",
      "Epoch 54/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.5341 - accuracy: 0.8589 - val_loss: 0.5669 - val_accuracy: 0.8428 - lr: 2.0000e-04\n",
      "Epoch 55/100\n",
      "781/781 [==============================] - 396s 508ms/step - loss: 0.5293 - accuracy: 0.8604 - val_loss: 0.5566 - val_accuracy: 0.8507 - lr: 2.0000e-04\n",
      "Epoch 56/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.5276 - accuracy: 0.8599 - val_loss: 0.6220 - val_accuracy: 0.8263 - lr: 2.0000e-04\n",
      "Epoch 57/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.5192 - accuracy: 0.8625 - val_loss: 0.5760 - val_accuracy: 0.8435 - lr: 2.0000e-04\n",
      "Epoch 58/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.5229 - accuracy: 0.8606 - val_loss: 0.5412 - val_accuracy: 0.8529 - lr: 2.0000e-04\n",
      "Epoch 59/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.5170 - accuracy: 0.8621 - val_loss: 0.5627 - val_accuracy: 0.8440 - lr: 2.0000e-04\n",
      "Epoch 60/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.5165 - accuracy: 0.8628 - val_loss: 0.5630 - val_accuracy: 0.8470 - lr: 2.0000e-04\n",
      "Epoch 61/100\n",
      "781/781 [==============================] - 112s 143ms/step - loss: 0.5152 - accuracy: 0.8631 - val_loss: 0.5811 - val_accuracy: 0.8398 - lr: 2.0000e-04\n",
      "Epoch 62/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.4970 - accuracy: 0.8673 - val_loss: 0.5525 - val_accuracy: 0.8534 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4957 - accuracy: 0.8686 - val_loss: 0.5762 - val_accuracy: 0.8426 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4865 - accuracy: 0.8716 - val_loss: 0.5470 - val_accuracy: 0.8512 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4884 - accuracy: 0.8710 - val_loss: 0.5605 - val_accuracy: 0.8453 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4939 - accuracy: 0.8694 - val_loss: 0.5447 - val_accuracy: 0.8521 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4839 - accuracy: 0.8719 - val_loss: 0.5486 - val_accuracy: 0.8538 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4868 - accuracy: 0.8707 - val_loss: 0.5654 - val_accuracy: 0.8450 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "781/781 [==============================] - 33s 42ms/step - loss: 0.4844 - accuracy: 0.8713 - val_loss: 0.5433 - val_accuracy: 0.8505 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4784 - accuracy: 0.8712 - val_loss: 0.5321 - val_accuracy: 0.8550 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "781/781 [==============================] - 32s 41ms/step - loss: 0.4775 - accuracy: 0.8727 - val_loss: 0.5326 - val_accuracy: 0.8537 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4821 - accuracy: 0.8715 - val_loss: 0.5407 - val_accuracy: 0.8529 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.4768 - accuracy: 0.8737 - val_loss: 0.5569 - val_accuracy: 0.8455 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4741 - accuracy: 0.8737 - val_loss: 0.5370 - val_accuracy: 0.8542 - lr: 1.0000e-04\n",
      "Epoch 75/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4737 - accuracy: 0.8748 - val_loss: 0.5538 - val_accuracy: 0.8448 - lr: 1.0000e-04\n",
      "Epoch 76/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4704 - accuracy: 0.8752 - val_loss: 0.5433 - val_accuracy: 0.8537 - lr: 1.0000e-04\n",
      "Epoch 77/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4707 - accuracy: 0.8747 - val_loss: 0.5348 - val_accuracy: 0.8509 - lr: 1.0000e-04\n",
      "Epoch 78/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4732 - accuracy: 0.8736 - val_loss: 0.5580 - val_accuracy: 0.8454 - lr: 1.0000e-04\n",
      "Epoch 79/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4650 - accuracy: 0.8759 - val_loss: 0.5734 - val_accuracy: 0.8401 - lr: 1.0000e-04\n",
      "Epoch 80/100\n",
      "781/781 [==============================] - 30s 38ms/step - loss: 0.4738 - accuracy: 0.8729 - val_loss: 0.5613 - val_accuracy: 0.8473 - lr: 1.0000e-04\n",
      "Epoch 81/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4690 - accuracy: 0.8736 - val_loss: 0.5367 - val_accuracy: 0.8532 - lr: 1.0000e-04\n",
      "Epoch 82/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4579 - accuracy: 0.8773 - val_loss: 0.5315 - val_accuracy: 0.8560 - lr: 5.0000e-05\n",
      "Epoch 83/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4610 - accuracy: 0.8772 - val_loss: 0.5310 - val_accuracy: 0.8551 - lr: 5.0000e-05\n",
      "Epoch 84/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4612 - accuracy: 0.8759 - val_loss: 0.5482 - val_accuracy: 0.8492 - lr: 5.0000e-05\n",
      "Epoch 85/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4575 - accuracy: 0.8799 - val_loss: 0.5299 - val_accuracy: 0.8549 - lr: 5.0000e-05\n",
      "Epoch 86/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4507 - accuracy: 0.8802 - val_loss: 0.5399 - val_accuracy: 0.8513 - lr: 5.0000e-05\n",
      "Epoch 87/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4497 - accuracy: 0.8799 - val_loss: 0.5462 - val_accuracy: 0.8519 - lr: 5.0000e-05\n",
      "Epoch 88/100\n",
      "781/781 [==============================] - 31s 39ms/step - loss: 0.4453 - accuracy: 0.8806 - val_loss: 0.5269 - val_accuracy: 0.8566 - lr: 5.0000e-05\n",
      "Epoch 89/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4539 - accuracy: 0.8786 - val_loss: 0.5452 - val_accuracy: 0.8491 - lr: 5.0000e-05\n",
      "Epoch 90/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4526 - accuracy: 0.8791 - val_loss: 0.5315 - val_accuracy: 0.8529 - lr: 5.0000e-05\n",
      "Epoch 91/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4489 - accuracy: 0.8802 - val_loss: 0.5370 - val_accuracy: 0.8520 - lr: 5.0000e-05\n",
      "Epoch 92/100\n",
      "781/781 [==============================] - 30s 39ms/step - loss: 0.4495 - accuracy: 0.8801 - val_loss: 0.5464 - val_accuracy: 0.8489 - lr: 5.0000e-05\n",
      "Epoch 93/100\n",
      "781/781 [==============================] - 48s 62ms/step - loss: 0.4461 - accuracy: 0.8799 - val_loss: 0.5248 - val_accuracy: 0.8569 - lr: 5.0000e-05\n",
      "Epoch 94/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4411 - accuracy: 0.8816 - val_loss: 0.5396 - val_accuracy: 0.8509 - lr: 5.0000e-05\n",
      "Epoch 95/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4408 - accuracy: 0.8816 - val_loss: 0.5367 - val_accuracy: 0.8525 - lr: 5.0000e-05\n",
      "Epoch 96/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4447 - accuracy: 0.8811 - val_loss: 0.5348 - val_accuracy: 0.8519 - lr: 5.0000e-05\n",
      "Epoch 97/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4463 - accuracy: 0.8805 - val_loss: 0.5362 - val_accuracy: 0.8525 - lr: 5.0000e-05\n",
      "Epoch 98/100\n",
      "781/781 [==============================] - 372s 476ms/step - loss: 0.4432 - accuracy: 0.8816 - val_loss: 0.5398 - val_accuracy: 0.8508 - lr: 5.0000e-05\n",
      "Epoch 99/100\n",
      "781/781 [==============================] - 32s 40ms/step - loss: 0.4397 - accuracy: 0.8834 - val_loss: 0.5538 - val_accuracy: 0.8471 - lr: 5.0000e-05\n",
      "Epoch 100/100\n",
      "781/781 [==============================] - 31s 40ms/step - loss: 0.4387 - accuracy: 0.8832 - val_loss: 0.5378 - val_accuracy: 0.8518 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2984fb3a0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# increasing dropout after convolutional layers and fully connected layers\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.datasets import cifar10\n",
    "from keras import regularizers\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "print(\"........\")\n",
    "print(tf.test.gpu_device_name())\n",
    "print(\"........\")\n",
    "def lr_schedule(epoch):\n",
    "    lrate = 0.001\n",
    "    if epoch > 80:\n",
    "        lrate = 0.00005\n",
    "    elif epoch > 60:\n",
    "        lrate = 0.0001\n",
    "    elif epoch > 40:\n",
    "        lrate = 0.0002\n",
    "    elif epoch > 20:\n",
    "        lrate = 0.0005        \n",
    "    return lrate\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = x_train / 255, x_test / 255\n",
    "\n",
    "#z-score\n",
    "\n",
    "\n",
    "num_classes = 10\n",
    "y_train = np_utils.to_categorical(y_train,num_classes)\n",
    "y_test = np_utils.to_categorical(y_test,num_classes)\n",
    "\n",
    "weight_decay = 1e-4\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay), input_shape=x_train.shape[1:]))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Conv2D(32, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Conv2D(64, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.3))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.4))\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.4))\n",
    "model.add(Conv2D(128, (3,3), padding='same', activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.4))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Conv2D(256, (3, 3), padding='same', activation='relu', kernel_initializer='he_uniform'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "#data augmentation\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=15,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1,\n",
    "    horizontal_flip=True,\n",
    "    )\n",
    "datagen.fit(x_train)\n",
    "\n",
    "#training\n",
    "batch_size = 64\n",
    "\n",
    "opt_rms = tf.keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt_rms, metrics=['accuracy'])\n",
    "model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\\\n",
    "                    steps_per_epoch=x_train.shape[0] // batch_size,epochs=100,\\\n",
    "                    verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}