02-speech-commands-mfcc-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isdir, join\n",
    "from tensorflow.keras import layers, models\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create list of all targets (minus background noise)\n",
    "dataset_path = 'C:\\\\PATH\\\\TO\\\\speech_commands_dataset'\n",
    "all_targets = all_targets = [name for name in listdir(dataset_path) if isdir(join(dataset_path, name))]\n",
    "all_targets.remove('_background_noise_')\n",
    "print(all_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Settings\n",
    "feature_sets_path = 'C:\\\\PATH\\\\TO\\\\feature_sets_directory'\n",
    "feature_sets_filename = 'all_targets_mfcc_sets.npz'\n",
    "model_filename = 'wake_word_stop_model.h5'\n",
    "wake_word = 'stop'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load feature sets\n",
    "feature_sets = np.load(join(feature_sets_path, feature_sets_filename))\n",
    "print(feature_sets.files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign feature sets\n",
    "x_train = feature_sets['x_train']\n",
    "y_train = feature_sets['y_train']\n",
    "x_val = feature_sets['x_val']\n",
    "y_val = feature_sets['y_val']\n",
    "x_test = feature_sets['x_test']\n",
    "y_test = feature_sets['y_test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at tensor dimensions\n",
    "print(x_train.shape)\n",
    "print(x_val.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Peek at labels\n",
    "print(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert ground truth arrays to one wake word (1) and 'other' (0)\n",
    "wake_word_index = all_targets.index(wake_word)\n",
    "y_train = np.equal(y_train, wake_word_index).astype('float64')\n",
    "y_val = np.equal(y_val, wake_word_index).astype('float64')\n",
    "y_test = np.equal(y_test, wake_word_index).astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Peek at labels after conversion\n",
    "print(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What percentage of 'stop' appear in validation labels\n",
    "print(sum(y_val) / len(y_val))\n",
    "print(1 - sum(y_val) / len(y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# View the dimensions of our input data\n",
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CNN for TF expects (batch, height, width, channels)\n",
    "# So we reshape the input tensors with a \"color\" channel of 1\n",
    "x_train = x_train.reshape(x_train.shape[0], \n",
    "                          x_train.shape[1], \n",
    "                          x_train.shape[2], \n",
    "                          1)\n",
    "x_val = x_val.reshape(x_val.shape[0], \n",
    "                      x_val.shape[1], \n",
    "                      x_val.shape[2], \n",
    "                      1)\n",
    "x_test = x_test.reshape(x_test.shape[0], \n",
    "                        x_test.shape[1], \n",
    "                        x_test.shape[2], \n",
    "                        1)\n",
    "print(x_train.shape)\n",
    "print(x_val.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input shape for CNN is size of MFCC of 1 sample\n",
    "sample_shape = x_test.shape[1:]\n",
    "print(sample_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build model\n",
    "# Based on: https://www.geeksforgeeks.org/python-image-classification-using-keras/\n",
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(32, \n",
    "                        (2, 2), \n",
    "                        activation='relu',\n",
    "                        input_shape=sample_shape))\n",
    "model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(32, (2, 2), activation='relu'))\n",
    "model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, (2, 2), activation='relu'))\n",
    "model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# Classifier\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(64, activation='relu'))\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add training parameters to model\n",
    "model.compile(loss='binary_crossentropy', \n",
    "              optimizer='rmsprop', \n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train\n",
    "history = model.fit(x_train, \n",
    "                    y_train, \n",
    "                    epochs=30, \n",
    "                    batch_size=100, \n",
    "                    validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot results\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model as a file\n",
    "models.save_model(model, model_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See which are 'stop'\n",
    "for idx, y in enumerate(y_test):\n",
    "    if y == 1:\n",
    "        print(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST: Load model and run it against test set\n",
    "model = models.load_model(model_filename)\n",
    "for i in range(100, 110):\n",
    "    print('Answer:', y_test[i], ' Prediction:', model.predict(np.expand_dims(x_test[i], 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate model with test set\n",
    "model.evaluate(x=x_test, y=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}