main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "# Small GPU Bugfix and all imports\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import tensorflow\n",
    "physical_devices = tensorflow.config.list_physical_devices('GPU')\n",
    "tensorflow.config.experimental.set_memory_growth(physical_devices[0], enable=True)\n",
    "\n",
    "import tensorflow_datasets as tfds\n",
    "import numpy as np\n",
    "from tensorflow import  numpy_function\n",
    "from tensorflow import convert_to_tensor\n",
    "from tensorflow import float32\n",
    "import tensorflow.keras as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.layers import Convolution1D, Flatten, Dense\n",
    "from tensorflow.keras.activations import softmax\n",
    "from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint, ReduceLROnPlateau\n",
    "from tensorflow.keras.models import load_model\n",
    "import librosa\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "class TensorBoardWithLR(TensorBoard):\n",
    "    def __init__(self, log_dir, **kwargs):  # add other arguments to __init__ if you need\n",
    "        super().__init__(log_dir=log_dir, **kwargs)\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        logs = logs or {}\n",
    "        logs.update({'lr': K.eval(self.model.optimizer.lr)})\n",
    "        super().on_epoch_end(epoch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Loading Dataset\n",
    "dataset = tfds.load(name=\"speech_commands\", split=\"train\")\n",
    "dataset = dataset.shuffle(1024)\n",
    "dataset_valid = tfds.load(name=\"speech_commands\", split=\"validation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import simpleaudio as sa\n",
    "for x in dataset.take(1):\n",
    "    audio = x['audio'].numpy()\n",
    "    print(x['label'])\n",
    "    audio = audio * (2**15 - 1) / np.max(np.abs(audio))\n",
    "    audio = audio.astype(np.int16)\n",
    "    play_obj = sa.play_buffer(audio, 1, 2, fs)\n",
    "    play_obj.wait_done()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Dataset Prep Functions\n",
    "@tensorflow.function\n",
    "def pad_spectrogram(spec, len):\n",
    "    zrs = tensorflow.zeros((spec.shape[0], len - spec.shape[1]), dtype=tensorflow.float32)\n",
    "    padded = tensorflow.concat([zrs, spec], 1)\n",
    "    return padded\n",
    "\n",
    "\n",
    "def to_spectrogram(x):\n",
    "    def mapping_function(audio, label):\n",
    "        S = librosa.feature.melspectrogram(audio.astype(np.float32), sr=16000, n_fft=160, hop_length=400,\n",
    "                                           n_mels=40)\n",
    "        S_DB = librosa.power_to_db(S, ref=np.max)\n",
    "        if S_DB.shape[1] != 41:\n",
    "            S_DB = pad_spectrogram(S_DB, 41)\n",
    "        S_DB = tensorflow.reshape(tensorflow.transpose(S_DB), (41, 40))\n",
    "        x = convert_to_tensor(S_DB)\n",
    "        y = convert_to_tensor(to_categorical(label, 12).astype(np.float32))\n",
    "        return x, y\n",
    "\n",
    "    audios, labels = numpy_function(mapping_function, [x['audio'], x['label']], [float32, float32])\n",
    "    audios.set_shape((41, 40))\n",
    "    labels.set_shape((12,))\n",
    "    return audios, labels\n",
    "\n",
    "def filter_predicate(x):\n",
    "    label = x['label']\n",
    "    return tf.backend.not_equal(label, 11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Apply Operations to Dataset\n",
    "dataset = dataset.filter(filter_predicate)\n",
    "dn = dataset.map(to_spectrogram)\n",
    "dataset_valid = dataset_valid.filter(filter_predicate)\n",
    "dataset_valid = dataset_valid.map(to_spectrogram)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Visualize Dataset\n",
    "from librosa.display import specshow\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "for x, y in dn.take(1):\n",
    "    specshow(x.numpy(), sr=16000, hop_length=400, x_axis='time', y_axis='mel')\n",
    "    plt.colorbar(format='%+2.0f dB')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Train Run prep\n",
    "RUN_NAME = 'conv1d_nounknowns_largermodel'\n",
    "tbcallback = TensorBoardWithLR(log_dir='logs/' + RUN_NAME)\n",
    "lr_reduction_callback = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)\n",
    "if 'saved_models' not in os.listdir('.'):\n",
    "    os.mkdir('saved_models')\n",
    "checkpointcallback = ModelCheckpoint('saved_models/'+RUN_NAME + '.h5', monitor='train-acc', save_best_only=False)\n",
    "dn = dn.batch(64)\n",
    "dataset_valid = dataset_valid.batch(64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Model Definition\n",
    "model = tf.Sequential()\n",
    "model.add(Convolution1D(64, 7, input_shape=(41, 40), activation='relu'))\n",
    "model.add(Convolution1D(32, 5, activation='relu'))\n",
    "model.add(Convolution1D(16, 3, activation='relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(12, activation=softmax))\n",
    "model.summary()\n",
    "model.compile(tf.optimizers.Adam(0.001), loss=tf.losses.categorical_crossentropy, metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Model Training\n",
    "model.fit(dn, callbacks=[tbcallback, checkpointcallback, lr_reduction_callback], epochs=512, validation_data=dataset_valid, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Model Testing\n",
    "conf_matrix = np.zeros((12, 12), np.uint64)\n",
    "for di in dn.take(100):\n",
    "    preds = model.predict(di[0])\n",
    "    for p, t in zip(np.argmax(preds, axis=-1), np.argmax(di[1], axis=-1)):\n",
    "        conf_matrix[p, t] += 1\n",
    "print(conf_matrix)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Load Model\n",
    "model = load_model('saved_models/'+RUN_NAME + '.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#Record a 1 second chunk of audio\n",
    "import sounddevice as sd\n",
    "from scipy.io.wavfile import write\n",
    "import matplotlib.pyplot as plt\n",
    "from librosa.display import specshow\n",
    "\n",
    "fs = 16000  # Sample rate\n",
    "seconds = 1  # Duration of recording\n",
    "print(\"START!\")\n",
    "myrecording = sd.rec(int(seconds * fs), samplerate=fs, channels=1)\n",
    "sd.wait()\n",
    "print(\"DONE!\")\n",
    "# plt.plot(myrecording[:, 0])\n",
    "write('output.wav', fs, myrecording)  # Save as WAV file\n",
    "plt.show()\n",
    "S = librosa.feature.melspectrogram(myrecording.astype(np.float32).reshape(16000,), sr=16000, n_fft=160, hop_length=400,\n",
    "                                           n_mels=40)\n",
    "S_DB = librosa.power_to_db(S, ref=np.max)\n",
    "# specshow(S_DB, sr=16000, hop_length=400, x_axis='time', y_axis='mel')\n",
    "labels = [\"Down\", \"Go\", \"Left\", \"No\", \"Off\", \"On\", \"Right\", \"Stop\", \"Up\", \"Yes\"]\n",
    "print(labels[np.argmax(model.predict(np.asarray([S_DB.T]))[0])])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}