CIFAR10_Keras_Convnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import os \n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_path = os.getcwd()\n",
    "dir_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpickle(file):\n",
    "    import pickle\n",
    "    with open(file, 'rb') as fo:\n",
    "        dict = pickle.load(fo, encoding='bytes')\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_1 = unpickle('datasets/cifar-10/data_batch_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_dataset_folder_path = 'datasets/cifar-10/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(cifar10_1.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cifar10_1[b'data']),len(cifar10_1[b'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_1[b'batch_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_Label_names = unpickle('datasets/cifar-10/batches.meta')\n",
    "cifar10_Label_names[b'label_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = []\n",
    "for x in cifar10_Label_names[b'label_names']:\n",
    "    label = x.decode()\n",
    "    label_names.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cfar10_batch(cifar10_dataset_folder_path, batch_id):\n",
    "    with open(cifar10_dataset_folder_path + '/data_batch_' + str(batch_id), mode='rb') as file:\n",
    "        # note the encoding type is 'latin1'\n",
    "        batch = pickle.load(file, encoding='latin1')\n",
    "        \n",
    "    features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)\n",
    "    labels = batch['labels']\n",
    "        \n",
    "    return features, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_stats(cifar10_dataset_folder_path, batch_id, sample_id):\n",
    "    features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_id)\n",
    "    \n",
    "    if not (0 <= sample_id < len(features)):\n",
    "        print('{} samples in batch {}.  {} is out of range.'.format(len(features), batch_id, sample_id))\n",
    "        return None\n",
    "\n",
    "    print('\\nStats of batch #{}:'.format(batch_id))\n",
    "    print('# of Samples: {}\\n'.format(len(features)))\n",
    "    \n",
    "    label_counts = dict(zip(*np.unique(labels, return_counts=True)))\n",
    "    for key, value in label_counts.items():\n",
    "        print('Label Counts of [{}]({}) : {}'.format(key, label_names[key].upper(), value))\n",
    "    \n",
    "    sample_image = features[sample_id]\n",
    "    sample_label = labels[sample_id]\n",
    "    \n",
    "    print('\\nExample of Image {}:'.format(sample_id))\n",
    "    print('Image - Min Value: {} Max Value: {}'.format(sample_image.min(), sample_image.max()))\n",
    "    print('Image - Shape: {}'.format(sample_image.shape))\n",
    "    print('Label - Label Id: {} Name: {}'.format(sample_label, label_names[sample_label]))\n",
    "    \n",
    "    plt.imshow(sample_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "# Explore the dataset\n",
    "batch_id = 3\n",
    "sample_id = 7000\n",
    "display_stats(cifar10_dataset_folder_path, batch_id, sample_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(x):\n",
    "\n",
    "    min_val = np.min(x)\n",
    "    max_val = np.max(x)\n",
    "    x = (x-min_val) / (max_val-min_val)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(x):\n",
    "\n",
    "    encoded = np.zeros((len(x), 10))\n",
    "    \n",
    "    for idx, val in enumerate(x):\n",
    "        encoded[idx][val] = 1\n",
    "    \n",
    "    return encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _preprocess_and_save(normalize, one_hot_encode, features, labels, filename):\n",
    "    features = normalize(features)\n",
    "    # one hot encoding only for categorical_crossentropy and categorical_accuracy\n",
    "    #labels = one_hot_encode(labels)\n",
    "\n",
    "    pickle.dump((features, labels), open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode):\n",
    "    n_batches = 5\n",
    "    valid_features = []\n",
    "    valid_labels = []\n",
    "    all_features = []\n",
    "    all_labels = []\n",
    "\n",
    "    for batch_i in range(1, n_batches + 1):\n",
    "        features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_i)\n",
    "        \n",
    "        # find index to be the point as validation data in the whole dataset of the batch (10%)\n",
    "        index_of_validation = int(len(features) * 0.1)\n",
    "\n",
    "        valid_features.extend(features[-index_of_validation:])\n",
    "        valid_labels.extend(labels[-index_of_validation:])\n",
    "        all_features.extend(features[:-index_of_validation])\n",
    "        all_labels.extend(labels[:-index_of_validation])\n",
    "\n",
    "    # preprocess the all stacked validation dataset\n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(valid_features), np.array(valid_labels),\n",
    "                         'datasets/preprocess_validation.p')\n",
    "                                 \n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(all_features), np.array(all_labels),\n",
    "                         'datasets/preprocess_all.p')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_features, valid_labels = pickle.load(open('datasets/preprocess_validation.p', mode='rb'))\n",
    "print(valid_features.dtype)\n",
    "print(valid_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, train_labels = pickle.load(open('datasets/preprocess_all.p', mode='rb'))\n",
    "print(train_features.dtype)\n",
    "print(train_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(target_size):\n",
    "\n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.layers.BatchNormalization(input_shape=(target_size, target_size, 3)))\n",
    "    model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same', activation='relu'))\n",
    "    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2,2)))\n",
    "    model.add(tf.keras.layers.Dropout(0.25))\n",
    "\n",
    "    model.add(tf.keras.layers.BatchNormalization(input_shape=(target_size, target_size, 3)))\n",
    "    model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))\n",
    "    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(tf.keras.layers.Dropout(0.25))\n",
    "\n",
    "    model.add(tf.keras.layers.BatchNormalization(input_shape=(target_size, target_size, 3)))\n",
    "    model.add(tf.keras.layers.Conv2D(256, (5, 5), padding='same', activation='relu'))\n",
    "    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2,2)))\n",
    "    model.add(tf.keras.layers.Dropout(0.25))\n",
    "\n",
    "    model.add(tf.keras.layers.Flatten())\n",
    "    model.add(tf.keras.layers.Dense(256))\n",
    "    model.add(tf.keras.layers.Activation('relu'))\n",
    "    model.add(tf.keras.layers.Dropout(0.5))\n",
    "    model.add(tf.keras.layers.Dense(10))\n",
    "    model.add(tf.keras.layers.Activation('softmax'))\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(32)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['sparse_categorical_accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainableLayers(model):\n",
    "    \n",
    "        for layer in model.layers:\n",
    "            layer.trainable = True  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainableLayers(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = tf.keras.callbacks.TensorBoard(log_dir=logdir, histogram_freq=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_all_batches(xEpochs):\n",
    "\n",
    "        model.fit(\n",
    "            train_features, train_labels,\n",
    "            epochs=xEpochs,\n",
    "            batch_size=32,\n",
    "            validation_data=(valid_features, valid_labels),\n",
    "            callbacks=[tensorboard] #[tensorboard, earlystopper]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For a better accuracy, increase the number of Epochs\n",
    "fit_all_batches(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_image = valid_features[3:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(sample_image)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_flat = y_pred.flatten()\n",
    "y_pred_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions():\n",
    "    \n",
    "    sample_image = valid_features[107:108]\n",
    "    print(sample_image.shape)\n",
    "    y_pred = model.predict(sample_image)\n",
    "    y_pred_flat = y_pred.flatten()\n",
    "    \n",
    "    results = np.column_stack((label_names, y_pred_flat))\n",
    "    print(results)\n",
    "    plt.imshow(valid_features[107])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_features[107].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nottrainableLayers(model):\n",
    "    \n",
    "        for layer in model.layers:\n",
    "            layer.trainable = False  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nottrainableLayers(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in model.layers:\n",
    "    print(l.name, l.trainable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras_file = \"models/keras_CIFAR_Seq_Sparse_model/keras_CIFAR_Seq_Sparse.h5\"\n",
    "model.save(keras_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to TensorFlow Lite model.\n",
    "converter = tf.lite.TFLiteConverter.from_keras_model_file(keras_file)\n",
    "converter.post_training_quantize=True\n",
    "tflite_model = converter.convert()\n",
    "open(\"models/keras_CIFAR_Seq_Sparse_model/keras_CIFAR_Seq_Sparse.tflite\", \"wb\").write(tflite_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}