01-speech-commands-mfcc-extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isdir, join\n",
    "import librosa\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import python_speech_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset path and view possible targets\n",
    "dataset_path = 'C:\\\\PATH\\\\TO\\\\speech_commands_dataset'\n",
    "for name in listdir(dataset_path):\n",
    "    if isdir(join(dataset_path, name)):\n",
    "        print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an all targets list\n",
    "all_targets = [name for name in listdir(dataset_path) if isdir(join(dataset_path, name))]\n",
    "print(all_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Leave off background noise set\n",
    "all_targets.remove('_background_noise_')\n",
    "print(all_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See how many files are in each\n",
    "num_samples = 0\n",
    "for target in all_targets:\n",
    "    print(len(listdir(join(dataset_path, target))))\n",
    "    num_samples += len(listdir(join(dataset_path, target)))\n",
    "print('Total samples:', num_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Settings\n",
    "target_list = all_targets\n",
    "feature_sets_file = 'all_targets_mfcc_sets.npz'\n",
    "perc_keep_samples = 1.0 # 1.0 is keep all samples\n",
    "val_ratio = 0.1\n",
    "test_ratio = 0.1\n",
    "sample_rate = 8000\n",
    "num_mfcc = 16\n",
    "len_mfcc = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create list of filenames along with ground truth vector (y)\n",
    "filenames = []\n",
    "y = []\n",
    "for index, target in enumerate(target_list):\n",
    "    print(join(dataset_path, target))\n",
    "    filenames.append(listdir(join(dataset_path, target)))\n",
    "    y.append(np.ones(len(filenames[index])) * index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check ground truth Y vector\n",
    "print(y)\n",
    "for item in y:\n",
    "    print(len(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Flatten filename and y vectors\n",
    "filenames = [item for sublist in filenames for item in sublist]\n",
    "y = [item for sublist in y for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Associate filenames with true output and shuffle\n",
    "filenames_y = list(zip(filenames, y))\n",
    "random.shuffle(filenames_y)\n",
    "filenames, y = zip(*filenames_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only keep the specified number of samples (shorter extraction/training)\n",
    "print(len(filenames))\n",
    "filenames = filenames[:int(len(filenames) * perc_keep_samples)]\n",
    "print(len(filenames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate validation and test set sizes\n",
    "val_set_size = int(len(filenames) * val_ratio)\n",
    "test_set_size = int(len(filenames) * test_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Break dataset apart into train, validation, and test sets\n",
    "filenames_val = filenames[:val_set_size]\n",
    "filenames_test = filenames[val_set_size:(val_set_size + test_set_size)]\n",
    "filenames_train = filenames[(val_set_size + test_set_size):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Break y apart into train, validation, and test sets\n",
    "y_orig_val = y[:val_set_size]\n",
    "y_orig_test = y[val_set_size:(val_set_size + test_set_size)]\n",
    "y_orig_train = y[(val_set_size + test_set_size):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: Create MFCC from given path\n",
    "def calc_mfcc(path):\n",
    "    \n",
    "    # Load wavefile\n",
    "    signal, fs = librosa.load(path, sr=sample_rate)\n",
    "    \n",
    "    # Create MFCCs from sound clip\n",
    "    mfccs = python_speech_features.base.mfcc(signal, \n",
    "                                            samplerate=fs,\n",
    "                                            winlen=0.256,\n",
    "                                            winstep=0.050,\n",
    "                                            numcep=num_mfcc,\n",
    "                                            nfilt=26,\n",
    "                                            nfft=2048,\n",
    "                                            preemph=0.0,\n",
    "                                            ceplifter=0,\n",
    "                                            appendEnergy=False,\n",
    "                                            winfunc=np.hanning)\n",
    "    return mfccs.transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST: Construct test set by computing MFCC of each WAV file\n",
    "prob_cnt = 0\n",
    "x_test = []\n",
    "y_test = []\n",
    "for index, filename in enumerate(filenames_train):\n",
    "    \n",
    "    # Stop after 500\n",
    "    if index >= 500:\n",
    "        break\n",
    "    \n",
    "    # Create path from given filename and target item\n",
    "    path = join(dataset_path, target_list[int(y_orig_train[index])], \n",
    "                filename)\n",
    "    \n",
    "    # Create MFCCs\n",
    "    mfccs = calc_mfcc(path)\n",
    "    \n",
    "    if mfccs.shape[1] == len_mfcc:\n",
    "        x_test.append(mfccs)\n",
    "        y_test.append(y_orig_train[index])\n",
    "    else:\n",
    "        print('Dropped:', index, mfccs.shape)\n",
    "        prob_cnt += 1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('% of problematic samples:', prob_cnt / 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST: Test shorter MFCC\n",
    "# !pip install playsound\n",
    "from playsound import playsound\n",
    "\n",
    "idx = 13\n",
    "\n",
    "# Create path from given filename and target item\n",
    "path = join(dataset_path, target_list[int(y_orig_train[idx])], \n",
    "            filenames_train[idx])\n",
    "\n",
    "# Create MFCCs\n",
    "mfccs = calc_mfcc(path)\n",
    "print(\"MFCCs:\", mfccs)\n",
    "\n",
    "# Plot MFCC\n",
    "fig = plt.figure()\n",
    "plt.imshow(mfccs, cmap='inferno', origin='lower')\n",
    "\n",
    "# TEST: Play problem sounds\n",
    "print(target_list[int(y_orig_train[idx])])\n",
    "playsound(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: Create MFCCs, keeping only ones of desired length\n",
    "def extract_features(in_files, in_y):\n",
    "    prob_cnt = 0\n",
    "    out_x = []\n",
    "    out_y = []\n",
    "        \n",
    "    for index, filename in enumerate(in_files):\n",
    "    \n",
    "        # Create path from given filename and target item\n",
    "        path = join(dataset_path, target_list[int(in_y[index])], \n",
    "                    filename)\n",
    "        \n",
    "        # Check to make sure we're reading a .wav file\n",
    "        if not path.endswith('.wav'):\n",
    "            continue\n",
    "\n",
    "        # Create MFCCs\n",
    "        mfccs = calc_mfcc(path)\n",
    "\n",
    "        # Only keep MFCCs with given length\n",
    "        if mfccs.shape[1] == len_mfcc:\n",
    "            out_x.append(mfccs)\n",
    "            out_y.append(in_y[index])\n",
    "        else:\n",
    "            print('Dropped:', index, mfccs.shape)\n",
    "            prob_cnt += 1\n",
    "            \n",
    "    return out_x, out_y, prob_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create train, validation, and test sets\n",
    "x_train, y_train, prob = extract_features(filenames_train, \n",
    "                                          y_orig_train)\n",
    "print('Removed percentage:', prob / len(y_orig_train))\n",
    "x_val, y_val, prob = extract_features(filenames_val, y_orig_val)\n",
    "print('Removed percentage:', prob / len(y_orig_val))\n",
    "x_test, y_test, prob = extract_features(filenames_test, y_orig_test)\n",
    "print('Removed percentage:', prob / len(y_orig_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save features and truth vector (y) sets to disk\n",
    "np.savez(feature_sets_file, \n",
    "         x_train=x_train, \n",
    "         y_train=y_train, \n",
    "         x_val=x_val, \n",
    "         y_val=y_val, \n",
    "         x_test=x_test, \n",
    "         y_test=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST: Load features\n",
    "feature_sets = np.load(feature_sets_file)\n",
    "feature_sets.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(feature_sets['x_train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(feature_sets['y_val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}