AttributionAnalysis_De.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "roshdl_R_SVy"
      },
      "source": [
        "# <font color=\"00ff00\">  **Statistical Analysis for Accented Speech recognition**</font>\n",
        "<font color=white> **evaluates the performance of an ASR model regarding accented speech**</font> \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kpDghOOKRF9n"
      },
      "source": [
        "# <font color=\"00ff00\">  **Statistical Analysis for Accented Speech recognition**</font>\n",
        "<font color=white> **evaluates the performance of an ASR model regarding accented speech**</font> \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OTQqnW3mRF9p"
      },
      "source": [
        "## <font color=\"00ff00\">  **Analyze results from Carlos’ model**</font>\n",
        "<font color=white> **Using Gradient-based techniques:**</font> "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BaIxeKyVRF9p"
      },
      "source": [
        "## <font color=\"00ff00\">  **3- Gradient-based Analysis**</font>\n",
        "<font color=white> **import Nemo Toolkit to use** </font> \n",
        "\n",
        "\n",
        " <font color=\"00ff00\">  **STT De Conformer-Transducer Large**</font>\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HwkaB-qW3AUn",
        "outputId": "ba87dc93-cb89-4c1b-9be2-04c0335187c3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting wget\n",
            "  Downloading wget-3.2.zip (10 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9674 sha256=a9224ad8c73893bb74da18e9fff91a53fa3409f4c7dbf834bd3ea784fbe3590d\n",
            "  Stored in directory: /root/.cache/pip/wheels/bd/a8/c3/3cf2c14a1837a4e04bd98631724e81f33f462d86a1d895fae0\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "libsndfile1 is already the newest version (1.0.28-7ubuntu0.1).\n",
            "ffmpeg is already the newest version (7:4.2.7-0ubuntu0.1).\n",
            "The following additional packages will be installed:\n",
            "  libopencore-amrnb0 libopencore-amrwb0 libsox-fmt-alsa libsox-fmt-base\n",
            "  libsox3\n",
            "Suggested packages:\n",
            "  libsox-fmt-all\n",
            "The following NEW packages will be installed:\n",
            "  libopencore-amrnb0 libopencore-amrwb0 libsox-fmt-alsa libsox-fmt-base\n",
            "  libsox3 sox\n",
            "0 upgraded, 6 newly installed, 0 to remove and 19 not upgraded.\n",
            "Need to get 513 kB of archives.\n",
            "After this operation, 1,564 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu focal/universe amd64 libopencore-amrnb0 amd64 0.1.5-1 [94.8 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu focal/universe amd64 libopencore-amrwb0 amd64 0.1.5-1 [49.1 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu focal/universe amd64 libsox3 amd64 14.4.2+git20190427-2 [226 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu focal/universe amd64 libsox-fmt-alsa amd64 14.4.2+git20190427-2 [10.5 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 libsox-fmt-base amd64 14.4.2+git20190427-2 [31.5 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 sox amd64 14.4.2+git20190427-2 [102 kB]\n",
            "Fetched 513 kB in 1s (748 kB/s)\n",
            "Selecting previously unselected package libopencore-amrnb0:amd64.\n",
            "(Reading database ... 128208 files and directories currently installed.)\n",
            "Preparing to unpack .../0-libopencore-amrnb0_0.1.5-1_amd64.deb ...\n",
            "Unpacking libopencore-amrnb0:amd64 (0.1.5-1) ...\n",
            "Selecting previously unselected package libopencore-amrwb0:amd64.\n",
            "Preparing to unpack .../1-libopencore-amrwb0_0.1.5-1_amd64.deb ...\n",
            "Unpacking libopencore-amrwb0:amd64 (0.1.5-1) ...\n",
            "Selecting previously unselected package libsox3:amd64.\n",
            "Preparing to unpack .../2-libsox3_14.4.2+git20190427-2_amd64.deb ...\n",
            "Unpacking libsox3:amd64 (14.4.2+git20190427-2) ...\n",
            "Selecting previously unselected package libsox-fmt-alsa:amd64.\n",
            "Preparing to unpack .../3-libsox-fmt-alsa_14.4.2+git20190427-2_amd64.deb ...\n",
            "Unpacking libsox-fmt-alsa:amd64 (14.4.2+git20190427-2) ...\n",
            "Selecting previously unselected package libsox-fmt-base:amd64.\n",
            "Preparing to unpack .../4-libsox-fmt-base_14.4.2+git20190427-2_amd64.deb ...\n",
            "Unpacking libsox-fmt-base:amd64 (14.4.2+git20190427-2) ...\n",
            "Selecting previously unselected package sox.\n",
            "Preparing to unpack .../5-sox_14.4.2+git20190427-2_amd64.deb ...\n",
            "Unpacking sox (14.4.2+git20190427-2) ...\n",
            "Setting up libsox3:amd64 (14.4.2+git20190427-2) ...\n",
            "Setting up libopencore-amrwb0:amd64 (0.1.5-1) ...\n",
            "Setting up libsox-fmt-alsa:amd64 (14.4.2+git20190427-2) ...\n",
            "Setting up libopencore-amrnb0:amd64 (0.1.5-1) ...\n",
            "Setting up libsox-fmt-base:amd64 (14.4.2+git20190427-2) ...\n",
            "Setting up sox (14.4.2+git20190427-2) ...\n",
            "Processing triggers for libc-bin (2.31-0ubuntu9.9) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n",
            "Processing triggers for mime-support (3.64ubuntu1) ...\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: text-unidecode in /usr/local/lib/python3.8/dist-packages (1.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting nemo_toolkit[all]\n",
            "  Cloning https://github.com/NVIDIA/NeMo.git (to revision main) to /tmp/pip-install-09c3f14h/nemo-toolkit_43361fc72cd74c12b6548caebb40c47a\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/NVIDIA/NeMo.git /tmp/pip-install-09c3f14h/nemo-toolkit_43361fc72cd74c12b6548caebb40c47a\n",
            "  Resolved https://github.com/NVIDIA/NeMo.git to commit 89b073e0fee6678788815166ace6b04b0289ad23\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting setuptools==59.5.0\n",
            "  Downloading setuptools-59.5.0-py3-none-any.whl (952 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m952.4/952.4 KB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wget in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.2)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.56.4)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (2.8.2)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.15.0)\n",
            "Collecting onnx>=1.7.0\n",
            "  Downloading onnx-1.13.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.5/13.5 MB\u001b[0m \u001b[31m74.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: text-unidecode in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.3)\n",
            "Collecting huggingface-hub\n",
            "  Downloading huggingface_hub-0.12.1-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ruamel.yaml\n",
            "  Downloading ruamel.yaml-0.17.21-py3-none-any.whl (109 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.5/109.5 KB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.13.1+cu116)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.0.2)\n",
            "Requirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.22 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.22.4)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (2.11.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (23.0)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.7.3)\n",
            "Collecting kaldiio\n",
            "  Downloading kaldiio-2.17.2.tar.gz (24 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting kaldi-python-io\n",
            "  Downloading kaldi-python-io-1.2.2.tar.gz (8.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting jiwer\n",
            "  Downloading jiwer-2.5.1-py3-none-any.whl (15 kB)\n",
            "Collecting sacremoses>=0.0.43\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m880.6/880.6 KB\u001b[0m \u001b[31m54.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: matplotlib>=3.3.2 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.5.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (1.3.5)\n",
            "Requirement already satisfied: librosa in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.8.1)\n",
            "Collecting sentence-transformers\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 KB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.13.10-py3-none-any.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m59.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers>=4.0.1\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m86.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: editdistance in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.5.3)\n",
            "Collecting flask-restful\n",
            "  Downloading Flask_RESTful-0.3.9-py2.py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: marshmallow in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.19.0)\n",
            "Collecting pyannote.metrics\n",
            "  Downloading pyannote.metrics-3.2.1-py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.4/51.4 KB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchmetrics>=0.11.0\n",
            "  Downloading torchmetrics-0.11.1-py3-none-any.whl (517 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m517.2/517.2 KB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting boto3\n",
            "  Downloading boto3-1.26.80-py3-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.7/132.7 KB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydub\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Collecting fasttext\n",
            "  Downloading fasttext-0.9.2.tar.gz (68 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m68.8/68.8 KB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting omegaconf<2.3,>=2.2\n",
            "  Downloading omegaconf-2.2.3-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.3/79.3 KB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytest-runner\n",
            "  Using cached pytest_runner-6.0.0-py3-none-any.whl (7.2 kB)\n",
            "Collecting isort<6.0.0,>5.1.0\n",
            "  Downloading isort-5.12.0-py3-none-any.whl (91 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.2/91.2 KB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting click==8.0.2\n",
            "  Downloading click-8.0.2-py3-none-any.whl (97 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.6/97.6 KB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jieba in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.42.1)\n",
            "Requirement already satisfied: gdown in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (4.4.0)\n",
            "Collecting gradio\n",
            "  Downloading gradio-3.19.1-py3-none-any.whl (14.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.2/14.2 MB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting einops\n",
            "  Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 KB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytorch-lightning>=1.9.0\n",
            "  Downloading pytorch_lightning-1.9.3-py3-none-any.whl (826 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m826.4/826.4 KB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sphinx in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.5.4)\n",
            "Collecting ftfy\n",
            "  Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 KB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pypinyin\n",
            "  Downloading pypinyin-0.48.0-py2.py3-none-any.whl (1.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m67.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: soundfile in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.12.1)\n",
            "Collecting ijson\n",
            "  Downloading ijson-3.2.0.post0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.8/114.8 KB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting youtokentome>=1.0.5\n",
            "  Downloading youtokentome-1.0.6-cp38-cp38-manylinux2010_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hydra-core<1.3,>=1.2.0\n",
            "  Downloading hydra_core-1.2.0-py3-none-any.whl (151 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m151.1/151.1 KB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyannote.core\n",
            "  Downloading pyannote.core-5.0.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.5/58.5 KB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sacrebleu[ja]\n",
            "  Downloading sacrebleu-2.3.1-py3-none-any.whl (118 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 KB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting texterrors\n",
            "  Downloading texterrors-0.4.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m58.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting parameterized\n",
            "  Downloading parameterized-0.8.1-py2.py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: nltk>=3.6.5 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.7)\n",
            "Collecting pyyaml<6\n",
            "  Downloading PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl (662 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m662.4/662.4 KB\u001b[0m \u001b[31m42.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sphinxcontrib-bibtex\n",
            "  Downloading sphinxcontrib_bibtex-2.5.0-py3-none-any.whl (39 kB)\n",
            "Collecting kornia\n",
            "  Downloading kornia-0.6.10-py2.py3-none-any.whl (612 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m612.0/612.0 KB\u001b[0m \u001b[31m44.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencc\n",
            "  Downloading OpenCC-1.1.6-cp38-cp38-manylinux1_x86_64.whl (778 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m778.2/778.2 KB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece<1.0.0\n",
            "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m51.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting faiss-cpu\n",
            "  Downloading faiss_cpu-1.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m49.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting braceexpand\n",
            "  Downloading braceexpand-0.1.7-py2.py3-none-any.whl (5.9 kB)\n",
            "Collecting sox\n",
            "  Downloading sox-1.4.1-py2.py3-none-any.whl (39 kB)\n",
            "Collecting webdataset<=0.1.62,>=0.1.48\n",
            "  Downloading webdataset-0.1.62-py3-none-any.whl (32 kB)\n",
            "Collecting pangu\n",
            "  Downloading pangu-4.0.6.1-py3-none-any.whl (6.4 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.7 in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (0.8.10)\n",
            "Collecting nemo-text-processing==0.1.6rc0\n",
            "  Downloading nemo_text_processing-0.1.6rc0-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m73.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: inflect in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (2.1.0)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (7.7.1)\n",
            "Collecting pypinyin-dict\n",
            "  Downloading pypinyin_dict-0.5.0-py2.py3-none-any.whl (9.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m77.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: h5py in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.1.0)\n",
            "Collecting black==19.10b0\n",
            "  Downloading black-19.10b0-py36-none-any.whl (97 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.5/97.5 KB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.8/dist-packages (from nemo_toolkit[all]) (3.6.4)\n",
            "Collecting rapidfuzz\n",
            "  Downloading rapidfuzz-2.13.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m54.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting progress>=1.5\n",
            "  Downloading progress-1.6.tar.gz (7.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting librosa\n",
            "  Downloading librosa-0.10.0-py3-none-any.whl (252 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.9/252.9 KB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting g2p-en\n",
            "  Downloading g2p_en-2.1.0-py3-none-any.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting textdistance>=4.1.5\n",
            "  Downloading textdistance-4.5.0-py3-none-any.whl (31 kB)\n",
            "Collecting attrdict\n",
            "  Downloading attrdict-2.0.1-py2.py3-none-any.whl (9.9 kB)\n",
            "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.8/dist-packages (from black==19.10b0->nemo_toolkit[all]) (22.2.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.8/dist-packages (from black==19.10b0->nemo_toolkit[all]) (2022.6.2)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.8/dist-packages (from black==19.10b0->nemo_toolkit[all]) (1.4.4)\n",
            "Collecting typed-ast>=1.4.0\n",
            "  Downloading typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (897 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m897.2/897.2 KB\u001b[0m \u001b[31m44.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pathspec<1,>=0.6\n",
            "  Downloading pathspec-0.11.0-py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: toml>=0.9.4 in /usr/local/lib/python3.8/dist-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.2)\n",
            "Collecting pynini\n",
            "  Downloading pynini-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (161.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m161.5/161.5 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (from nemo-text-processing==0.1.6rc0->nemo_toolkit[all]) (1.2.0)\n",
            "Collecting cdifflib\n",
            "  Downloading cdifflib-1.2.6.tar.gz (11 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: Cython>=0.29 in /usr/local/lib/python3.8/dist-packages (from pynini->nemo-text-processing==0.1.6rc0->nemo_toolkit[all]) (0.29.33)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.8/dist-packages (from hydra-core<1.3,>=1.2.0->nemo_toolkit[all]) (5.12.0)\n",
            "Collecting antlr4-python3-runtime==4.9.*\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 KB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting levenshtein==0.20.2\n",
            "  Downloading Levenshtein-0.20.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m64.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.8/dist-packages (from librosa->nemo_toolkit[all]) (4.4.2)\n",
            "Requirement already satisfied: typing-extensions>=4.1.1 in /usr/local/lib/python3.8/dist-packages (from librosa->nemo_toolkit[all]) (4.5.0)\n",
            "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.8/dist-packages (from librosa->nemo_toolkit[all]) (3.0.0)\n",
            "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.8/dist-packages (from librosa->nemo_toolkit[all]) (1.0.4)\n",
            "Collecting lazy-loader>=0.1\n",
            "  Downloading lazy_loader-0.1-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.8/dist-packages (from librosa->nemo_toolkit[all]) (1.6.0)\n",
            "Collecting soxr>=0.3.2\n",
            "  Downloading soxr-0.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m64.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (3.0.9)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (4.38.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (1.4.4)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (8.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (0.11.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba->nemo_toolkit[all]) (6.0.0)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba->nemo_toolkit[all]) (0.39.1)\n",
            "Collecting protobuf<4,>=3.20.2\n",
            "  Downloading protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m55.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil->nemo_toolkit[all]) (1.15.0)\n",
            "Collecting lightning-utilities>=0.6.0.post0\n",
            "  Downloading lightning_utilities-0.7.1-py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: fsspec[http]>2021.06.0 in /usr/local/lib/python3.8/dist-packages (from pytorch-lightning>=1.9.0->nemo_toolkit[all]) (2023.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn->nemo_toolkit[all]) (3.1.0)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.8/dist-packages (from soundfile->nemo_toolkit[all]) (1.15.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m84.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers>=4.0.1->nemo_toolkit[all]) (2.25.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers>=4.0.1->nemo_toolkit[all]) (3.9.0)\n",
            "Collecting s3transfer<0.7.0,>=0.6.0\n",
            "  Downloading s3transfer-0.6.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.6/79.6 KB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting botocore<1.30.0,>=1.29.80\n",
            "  Downloading botocore-1.29.80-py3-none-any.whl (10.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.5/10.5 MB\u001b[0m \u001b[31m67.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1\n",
            "  Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
            "Collecting pybind11>=2.2\n",
            "  Using cached pybind11-2.10.3-py3-none-any.whl (222 kB)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.8/dist-packages (from flask-restful->nemo_toolkit[all]) (2022.7.1)\n",
            "Requirement already satisfied: Flask>=0.8 in /usr/local/lib/python3.8/dist-packages (from flask-restful->nemo_toolkit[all]) (2.2.3)\n",
            "Collecting aniso8601>=0.82\n",
            "  Downloading aniso8601-9.0.1-py2.py3-none-any.whl (52 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 KB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.8/dist-packages (from ftfy->nemo_toolkit[all]) (0.2.6)\n",
            "Collecting distance>=0.1.3\n",
            "  Downloading Distance-0.1.3.tar.gz (180 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m180.3/180.3 KB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.8/dist-packages (from gdown->nemo_toolkit[all]) (4.6.3)\n",
            "Collecting python-multipart\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 KB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn\n",
            "  Downloading uvicorn-0.20.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.9/56.9 KB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.0\n",
            "  Downloading websockets-10.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 KB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy\n",
            "  Downloading ffmpy-0.3.0.tar.gz (4.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting httpx\n",
            "  Downloading httpx-0.23.3-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 KB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic in /usr/local/lib/python3.8/dist-packages (from gradio->nemo_toolkit[all]) (1.10.5)\n",
            "Collecting fastapi\n",
            "  Downloading fastapi-0.92.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.2/56.2 KB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mdit-py-plugins<=0.3.3\n",
            "  Downloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 KB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from gradio->nemo_toolkit[all]) (3.8.4)\n",
            "Collecting orjson\n",
            "  Downloading orjson-3.8.7-cp38-cp38-manylinux_2_28_x86_64.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.7/140.7 KB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycryptodome\n",
            "  Downloading pycryptodome-3.17-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m74.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting markdown-it-py[linkify]>=2.0.0\n",
            "  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 KB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiofiles\n",
            "  Downloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.8/dist-packages (from gradio->nemo_toolkit[all]) (2.1.2)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from gradio->nemo_toolkit[all]) (4.2.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from gradio->nemo_toolkit[all]) (3.1.2)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (3.0.5)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (7.9.0)\n",
            "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (0.2.0)\n",
            "Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (3.6.2)\n",
            "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (5.3.4)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.8/dist-packages (from ipywidgets->nemo_toolkit[all]) (5.7.1)\n",
            "Requirement already satisfied: sortedcontainers>=2.0.4 in /usr/local/lib/python3.8/dist-packages (from pyannote.core->nemo_toolkit[all]) (2.4.0)\n",
            "Requirement already satisfied: sympy>=1.1 in /usr/local/lib/python3.8/dist-packages (from pyannote.metrics->nemo_toolkit[all]) (1.7.1)\n",
            "Collecting pyannote.database>=4.0.1\n",
            "  Downloading pyannote.database-4.1.3-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 KB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docopt>=0.6.2\n",
            "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.8/dist-packages (from pytest->nemo_toolkit[all]) (1.11.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.8/dist-packages (from pytest->nemo_toolkit[all]) (0.7.1)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.8/dist-packages (from pytest->nemo_toolkit[all]) (1.4.1)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.8/dist-packages (from pytest->nemo_toolkit[all]) (9.0.0)\n",
            "Collecting ruamel.yaml.clib>=0.2.6\n",
            "  Downloading ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (555 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m555.3/555.3 KB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting portalocker\n",
            "  Downloading portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.8/dist-packages (from sacrebleu[ja]->nemo_toolkit[all]) (4.9.2)\n",
            "Collecting mecab-python3==1.0.5\n",
            "  Downloading mecab_python3-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (577 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m577.3/577.3 KB\u001b[0m \u001b[31m41.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipadic<2.0,>=1.0\n",
            "  Downloading ipadic-1.0.0.tar.gz (13.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.4/13.4 MB\u001b[0m \u001b[31m75.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.8/dist-packages (from sentence-transformers->nemo_toolkit[all]) (0.14.1+cu116)\n",
            "Requirement already satisfied: babel>=1.3 in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (2.11.0)\n",
            "Requirement already satisfied: sphinxcontrib-applehelp in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.0.4)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (0.7.13)\n",
            "Requirement already satisfied: sphinxcontrib-jsmath in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.0.1)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.4.1)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.1.5)\n",
            "Requirement already satisfied: sphinxcontrib-devhelp in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (2.6.1)\n",
            "Requirement already satisfied: sphinxcontrib-htmlhelp in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (2.0.1)\n",
            "Requirement already satisfied: sphinxcontrib-qthelp in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (1.0.3)\n",
            "Requirement already satisfied: docutils<0.17,>=0.12 in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (0.16)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.8/dist-packages (from sphinx->nemo_toolkit[all]) (2.2.0)\n",
            "Collecting pybtex-docutils>=1.0.0\n",
            "  Downloading pybtex_docutils-1.0.2-py3-none-any.whl (6.3 kB)\n",
            "Collecting pybtex>=0.24\n",
            "  Downloading pybtex-0.24.0-py2.py3-none-any.whl (561 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m561.4/561.4 KB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (1.4.0)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (2.2.3)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (0.38.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (1.8.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (0.6.1)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (1.51.3)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (2.16.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.8/dist-packages (from tensorboard->nemo_toolkit[all]) (3.4.1)\n",
            "Collecting plac\n",
            "  Downloading plac-1.3.5-py2.py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.8/dist-packages (from texterrors->nemo_toolkit[all]) (2.2.0)\n",
            "Collecting loguru\n",
            "  Downloading loguru-0.6.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 KB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.16.0-py2.py3-none-any.whl (184 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 KB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 KB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting setproctitle\n",
            "  Downloading setproctitle-1.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (31 kB)\n",
            "Collecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.8/dist-packages (from wandb->nemo_toolkit[all]) (5.4.8)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.8/dist-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.8/dist-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (0.12.0)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.8/dist-packages (from altair>=4.2.0->gradio->nemo_toolkit[all]) (0.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.8/dist-packages (from botocore<1.30.0,>=1.29.80->boto3->nemo_toolkit[all]) (1.26.14)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.8/dist-packages (from cffi>=1.0->soundfile->nemo_toolkit[all]) (2.21)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.8/dist-packages (from Flask>=0.8->flask-restful->nemo_toolkit[all]) (2.1.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.8.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (6.0.4)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (3.0.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio->nemo_toolkit[all]) (4.0.2)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 KB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (5.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (4.9)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->nemo_toolkit[all]) (1.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata->numba->nemo_toolkit[all]) (3.15.0)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.8/dist-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (6.1.12)\n",
            "Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.8/dist-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (6.2)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (2.0.10)\n",
            "Collecting jedi>=0.10\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pexpect in /usr/local/lib/python3.8/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (4.8.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.8/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.8/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.5)\n",
            "Collecting mdurl~=0.1\n",
            "  Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
            "Collecting linkify-it-py<3,>=1\n",
            "  Downloading linkify_it_py-2.0.0-py3-none-any.whl (19 kB)\n",
            "Requirement already satisfied: typer[all]>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.7.0)\n",
            "Collecting latexcodec>=1.0.4\n",
            "  Downloading latexcodec-2.0.1-py2.py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (4.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (2.10)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.8/dist-packages (from sympy>=1.1->pyannote.metrics->nemo_toolkit[all]) (1.2.1)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.8/dist-packages (from widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (6.3.0)\n",
            "Collecting starlette<0.26.0,>=0.25.0\n",
            "  Downloading starlette-0.25.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 KB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sniffio\n",
            "  Downloading sniffio-1.3.0-py3-none-any.whl (10 kB)\n",
            "Collecting httpcore<0.17.0,>=0.15.0\n",
            "  Downloading httpcore-0.16.3-py3-none-any.whl (69 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 KB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rfc3986[idna2008]<2,>=1.3\n",
            "  Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.8/dist-packages (from requests->transformers>=4.0.1->nemo_toolkit[all]) (1.7.1)\n",
            "Collecting h11>=0.8\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 KB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting smmap<6,>=3.0.1\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Collecting anyio<5.0,>=3.0\n",
            "  Downloading anyio-3.6.2-py3-none-any.whl (80 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.6/80.6 KB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.8/dist-packages (from jedi>=0.10->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.8.3)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio->nemo_toolkit[all]) (0.19.3)\n",
            "Collecting uc-micro-py\n",
            "  Downloading uc_micro_py-1.0.1-py3-none-any.whl (6.2 kB)\n",
            "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (21.3.0)\n",
            "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.16.0)\n",
            "Requirement already satisfied: Send2Trash>=1.5.0 in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (1.8.0)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (5.7.3)\n",
            "Requirement already satisfied: jupyter-core>=4.6.1 in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (5.2.0)\n",
            "Requirement already satisfied: pyzmq>=17 in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (23.2.1)\n",
            "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.13.3)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.8/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (6.5.4)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->nemo_toolkit[all]) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->nemo_toolkit[all]) (3.2.2)\n",
            "Collecting rich<13.0.0,>=10.11.0\n",
            "  Downloading rich-12.6.0-py3-none-any.whl (237 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m237.5/237.5 KB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting shellingham<2.0.0,>=1.3.0\n",
            "  Downloading shellingham-1.5.0.post1-py2.py3-none-any.whl (9.4 kB)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.8/dist-packages (from pexpect->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.0)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.8/dist-packages (from jupyter-core>=4.6.1->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (3.0.0)\n",
            "Collecting commonmark<0.10.0,>=0.9.0\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 KB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.8/dist-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (21.2.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.8.4)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.7.1)\n",
            "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.7.2)\n",
            "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (1.2.1)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (6.0.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (1.5.0)\n",
            "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.8/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.2.2)\n",
            "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.8/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (2.16.3)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.8/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->nemo_toolkit[all]) (0.5.1)\n",
            "Building wheels for collected packages: antlr4-python3-runtime, progress, sacremoses, fasttext, kaldi-python-io, kaldiio, nemo_toolkit, sentence-transformers, distance, docopt, ipadic, cdifflib, ffmpy, pathtools\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144575 sha256=c0aa1958ddfed03713a6f1d6741b3de60750c90a2fb09fe2d0db24093e9a7d15\n",
            "  Stored in directory: /root/.cache/pip/wheels/b1/a3/c2/6df046c09459b73cc9bb6c4401b0be6c47048baf9a1617c485\n",
            "  Building wheel for progress (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for progress: filename=progress-1.6-py3-none-any.whl size=9630 sha256=5b1e0d9928ced71287fd63c1d74ca9a008d1c313073069e57fe27262ed327045\n",
            "  Stored in directory: /root/.cache/pip/wheels/bb/01/5a/c916509df9b12c6465864251dbe826def8e31a16fa7da54f08\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=812a8160477e4da94fd870b4f001a060ed1df7ef06353f1b19a36e0396bb93d4\n",
            "  Stored in directory: /root/.cache/pip/wheels/82/ab/9b/c15899bf659ba74f623ac776e861cf2eb8608c1825ddec66a4\n",
            "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fasttext: filename=fasttext-0.9.2-cp38-cp38-linux_x86_64.whl size=4402300 sha256=b6dc972e37e5ea315103323736215247b53e5029863c59686d17bc0ab655aeca\n",
            "  Stored in directory: /root/.cache/pip/wheels/93/61/2a/c54711a91c418ba06ba195b1d78ff24fcaad8592f2a694ac94\n",
            "  Building wheel for kaldi-python-io (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaldi-python-io: filename=kaldi_python_io-1.2.2-py3-none-any.whl size=8970 sha256=502424955c9d0f66a6bba4ce08b447fc23dadafa7eccdefd6cf9883170b0f59c\n",
            "  Stored in directory: /root/.cache/pip/wheels/8c/da/46/4afb7e26376c33af41c3ec388d5b63d34d186f6df1545cac30\n",
            "  Building wheel for kaldiio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaldiio: filename=kaldiio-2.17.2-py3-none-any.whl size=24471 sha256=50e768b2c3d7d1fbeeb0d2d0c1d2d15758ad3596b2303162087b1f5985c5e698\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/b3/00/af2103b510836161326bb51e27795407b07fda6969d0ae5967\n",
            "  Building wheel for nemo_toolkit (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nemo_toolkit: filename=nemo_toolkit-1.16.0rc0-py3-none-any.whl size=2230615 sha256=82b145b3bb4f68032af6fdb22dba5ea439cf7c0f06853c346fc7a31ae9c16509\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-3w5ujuta/wheels/76/8c/14/f1ad178881c7340cf8e629686593904531249cf8b2de155d8c\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125938 sha256=acaa80f86060c429822117efd0d72ee44d554c0e61f812c9ba9bf357dc11d055\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/6f/8c/d88aec621f3f542d26fac0342bef5e693335d125f4e54aeffe\n",
            "  Building wheel for distance (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for distance: filename=Distance-0.1.3-py3-none-any.whl size=16275 sha256=4236b33cecfcfef92f27b5a39748119b451958dff23aac4d9300a5f0b078bfb1\n",
            "  Stored in directory: /root/.cache/pip/wheels/fb/a8/64/6edcab63ec51512a87cacf9b3563c711ad6b7b05d61b704493\n",
            "  Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13723 sha256=47f113e1c6415d5c82adf17d18ca20f6c882a76f61502ac4b58bed6e6dc12521\n",
            "  Stored in directory: /root/.cache/pip/wheels/56/ea/58/ead137b087d9e326852a851351d1debf4ada529b6ac0ec4e8c\n",
            "  Building wheel for ipadic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556723 sha256=55f330d283af7c8b1efca2831e3d6bcff5efdfe8060642ab12f00552653b6aa9\n",
            "  Stored in directory: /root/.cache/pip/wheels/45/b7/f5/a21e68db846eedcd00d69e37d60bab3f68eb20b1d99cdff652\n",
            "  Building wheel for cdifflib (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cdifflib: filename=cdifflib-1.2.6-cp38-cp38-linux_x86_64.whl size=37353 sha256=caf5489c269ed18123bf9d4e867f8621040c11d9751682a8514f17e296183a5c\n",
            "  Stored in directory: /root/.cache/pip/wheels/0f/ae/42/928075504d5f17237e6efae65604402e73598b83d1f4c77b0f\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.0-py3-none-any.whl size=4711 sha256=923deb694615d928f118cc29afa86e51e4961897bd4a0814caf0022d1355471f\n",
            "  Stored in directory: /root/.cache/pip/wheels/ff/5b/59/913b443e7369dc04b61f607a746b6f7d83fb65e2e19fcc958d\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=d6644bff250902cd7efa5ab7202742138d24ca397ddd9308b5fa02837d8ad3cc\n",
            "  Stored in directory: /root/.cache/pip/wheels/4c/8e/7e/72fbc243e1aeecae64a96875432e70d4e92f3d2d18123be004\n",
            "Successfully built antlr4-python3-runtime progress sacremoses fasttext kaldi-python-io kaldiio nemo_toolkit sentence-transformers distance docopt ipadic cdifflib ffmpy pathtools\n",
            "Installing collected packages: tokenizers, sentencepiece, rfc3986, pydub, progress, plac, pathtools, parameterized, pangu, opencc, mecab-python3, ipadic, ijson, ffmpy, faiss-cpu, docopt, distance, commonmark, cdifflib, braceexpand, antlr4-python3-runtime, aniso8601, websockets, webdataset, uc-micro-py, typed-ast, textdistance, soxr, sox, sniffio, smmap, shellingham, setuptools, setproctitle, sentry-sdk, ruamel.yaml.clib, rich, rapidfuzz, pyyaml, python-multipart, pytest-runner, pypinyin, pynini, pycryptodome, pybind11, protobuf, portalocker, pathspec, orjson, mdurl, loguru, lightning-utilities, lazy-loader, latexcodec, kaldiio, kaldi-python-io, jmespath, jedi, isort, h11, ftfy, einops, docker-pycreds, colorama, click, attrdict, aiofiles, youtokentome, uvicorn, torchmetrics, sacremoses, sacrebleu, ruamel.yaml, pypinyin-dict, pybtex, pyannote.core, onnx, omegaconf, markdown-it-py, linkify-it-py, levenshtein, kornia, huggingface-hub, gitdb, fasttext, botocore, black, anyio, transformers, texterrors, starlette, s3transfer, pybtex-docutils, mdit-py-plugins, librosa, jiwer, hydra-core, httpcore, GitPython, g2p-en, flask-restful, wandb, sphinxcontrib-bibtex, sentence-transformers, pytorch-lightning, pyannote.database, nemo-text-processing, httpx, fastapi, boto3, pyannote.metrics, nemo_toolkit, gradio\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.4.0\n",
            "    Uninstalling setuptools-57.4.0:\n",
            "      Successfully uninstalled setuptools-57.4.0\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 6.0\n",
            "    Uninstalling PyYAML-6.0:\n",
            "      Successfully uninstalled PyYAML-6.0\n",
            "  Attempting uninstall: protobuf\n",
            "    Found existing installation: protobuf 3.19.6\n",
            "    Uninstalling protobuf-3.19.6:\n",
            "      Successfully uninstalled protobuf-3.19.6\n",
            "  Attempting uninstall: click\n",
            "    Found existing installation: click 8.1.3\n",
            "    Uninstalling click-8.1.3:\n",
            "      Successfully uninstalled click-8.1.3\n",
            "  Attempting uninstall: librosa\n",
            "    Found existing installation: librosa 0.8.1\n",
            "    Uninstalling librosa-0.8.1:\n",
            "      Successfully uninstalled librosa-0.8.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.11.0 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed GitPython-3.1.31 aiofiles-23.1.0 aniso8601-9.0.1 antlr4-python3-runtime-4.9.3 anyio-3.6.2 attrdict-2.0.1 black-19.10b0 boto3-1.26.80 botocore-1.29.80 braceexpand-0.1.7 cdifflib-1.2.6 click-8.0.2 colorama-0.4.6 commonmark-0.9.1 distance-0.1.3 docker-pycreds-0.4.0 docopt-0.6.2 einops-0.6.0 faiss-cpu-1.7.3 fastapi-0.92.0 fasttext-0.9.2 ffmpy-0.3.0 flask-restful-0.3.9 ftfy-6.1.1 g2p-en-2.1.0 gitdb-4.0.10 gradio-3.19.1 h11-0.14.0 httpcore-0.16.3 httpx-0.23.3 huggingface-hub-0.12.1 hydra-core-1.2.0 ijson-3.2.0.post0 ipadic-1.0.0 isort-5.12.0 jedi-0.18.2 jiwer-2.5.1 jmespath-1.0.1 kaldi-python-io-1.2.2 kaldiio-2.17.2 kornia-0.6.10 latexcodec-2.0.1 lazy-loader-0.1 levenshtein-0.20.2 librosa-0.10.0 lightning-utilities-0.7.1 linkify-it-py-2.0.0 loguru-0.6.0 markdown-it-py-2.2.0 mdit-py-plugins-0.3.3 mdurl-0.1.2 mecab-python3-1.0.5 nemo-text-processing-0.1.6rc0 nemo_toolkit-1.16.0rc0 omegaconf-2.2.3 onnx-1.13.1 opencc-1.1.6 orjson-3.8.7 pangu-4.0.6.1 parameterized-0.8.1 pathspec-0.11.0 pathtools-0.1.2 plac-1.3.5 portalocker-2.7.0 progress-1.6 protobuf-3.20.3 pyannote.core-5.0.0 pyannote.database-4.1.3 pyannote.metrics-3.2.1 pybind11-2.10.3 pybtex-0.24.0 pybtex-docutils-1.0.2 pycryptodome-3.17 pydub-0.25.1 pynini-2.1.5 pypinyin-0.48.0 pypinyin-dict-0.5.0 pytest-runner-6.0.0 python-multipart-0.0.6 pytorch-lightning-1.9.3 pyyaml-5.4.1 rapidfuzz-2.13.7 rfc3986-1.5.0 rich-12.6.0 ruamel.yaml-0.17.21 ruamel.yaml.clib-0.2.7 s3transfer-0.6.0 sacrebleu-2.3.1 sacremoses-0.0.53 sentence-transformers-2.2.2 sentencepiece-0.1.97 sentry-sdk-1.16.0 setproctitle-1.3.2 setuptools-59.5.0 shellingham-1.5.0.post1 smmap-5.0.0 sniffio-1.3.0 sox-1.4.1 soxr-0.3.3 sphinxcontrib-bibtex-2.5.0 starlette-0.25.0 textdistance-4.5.0 texterrors-0.4.4 tokenizers-0.13.2 torchmetrics-0.11.1 transformers-4.26.1 typed-ast-1.5.4 uc-micro-py-1.0.1 uvicorn-0.20.0 wandb-0.13.10 webdataset-0.1.62 websockets-10.4 youtokentome-1.0.6\n"
          ]
        }
      ],
      "source": [
        "\"\"\"\n",
        "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n",
        "\n",
        "Instructions for setting up Colab are as follows:\n",
        "1. Open a new Python 3 notebook.\n",
        "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n",
        "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n",
        "4. Run this cell to set up dependencies.\n",
        "5. Restart the runtime (Runtime -> Restart Runtime) for any upgraded packages to take effect\n",
        "\"\"\"\n",
        "# If you're using Google Colab and not running locally, run this cell.\n",
        "\n",
        "## Install dependencies\n",
        "!pip install wget\n",
        "!apt-get install sox libsndfile1 ffmpeg\n",
        "!pip install text-unidecode\n",
        "!pip install matplotlib>=3.3.2\n",
        "\n",
        "## Install NeMo\n",
        "BRANCH = 'main'\n",
        "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[all]\n",
        "\n",
        "\"\"\"\n",
        "Remember to restart the runtime for the kernel to pick up any upgraded packages (e.g. matplotlib)!\n",
        "Alternatively, you can uncomment the exit() below to crash and restart the kernel, in the case\n",
        "that you want to use the \"Run All Cells\" (or similar) option.\n",
        "\"\"\"\n",
        "exit()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cydRZFRhurMx",
        "outputId": "077ebd64-dfd8-4ccf-c721-c0c04f93aef9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[NeMo W 2023-02-28 17:40:34 optimizers:54] Apex was not found. Using the lamb or fused_adam optimizer will error out.\n",
            "[NeMo W 2023-02-28 17:40:40 experimental:27] Module <class 'nemo.collections.asr.modules.audio_modules.SpectrogramToMultichannelFeatures'> is experimental, not ready for production and is not fully supported. Use at your own risk.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NeMo I 2023-02-28 17:40:42 cloud:68] Downloading from: https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_de_conformer_transducer_large/versions/1.5.0/files/stt_de_conformer_transducer_large.nemo to /root/.cache/torch/NeMo/NeMo_1.16.0rc0/stt_de_conformer_transducer_large/183f232a57704e78fd9b211a3e021f02/stt_de_conformer_transducer_large.nemo\n",
            "[NeMo I 2023-02-28 17:40:57 common:913] Instantiating model from pre-trained checkpoint\n",
            "[NeMo I 2023-02-28 17:41:07 mixins:170] Tokenizer SentencePieceTokenizer initialized with 1024 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[NeMo W 2023-02-28 17:41:09 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\n",
            "    Train config : \n",
            "    manifest_filepath: /data/train/tarred_audio_manifest.json\n",
            "    sample_rate: 16000\n",
            "    batch_size: 8\n",
            "    shuffle: true\n",
            "    num_workers: 4\n",
            "    pin_memory: true\n",
            "    use_start_end_token: true\n",
            "    trim_silence: false\n",
            "    max_duration: 20.0\n",
            "    min_duration: 0.1\n",
            "    shuffle_n: 2048\n",
            "    is_tarred: true\n",
            "    tarred_audio_filepaths: /data/train/audio__OP_0..1023_CL_.tar\n",
            "    \n",
            "[NeMo W 2023-02-28 17:41:09 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n",
            "    Validation config : \n",
            "    manifest_filepath:\n",
            "    - /data/voxpopuli_de/dev/voxpopuli_dev_manifest.json\n",
            "    - /data/voxpopuli_de/test/voxpopuli_test_manifest.json\n",
            "    - /data/mcv7.0_de/mcv7.0_dev_manifest_cleaned.json\n",
            "    - /data/mcv7.0_de/mcv7.0_test_manifest_cleaned.json\n",
            "    - /data/mls_de/mls_dev_manifest_cleaned.json\n",
            "    - /data/mls_de/mls_test_manifest_cleaned.json\n",
            "    sample_rate: 16000\n",
            "    batch_size: 8\n",
            "    shuffle: false\n",
            "    num_workers: 8\n",
            "    pin_memory: true\n",
            "    use_start_end_token: true\n",
            "    \n",
            "[NeMo W 2023-02-28 17:41:09 modelPT:174] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).\n",
            "    Test config : \n",
            "    manifest_filepath:\n",
            "    - /data/voxpopuli_de/test/voxpopuli_test_manifest.json\n",
            "    - /data/mcv7.0_de/mcv7.0_test_manifest_cleaned.json\n",
            "    - /data/mls_de/mls_test_manifest_cleaned.json\n",
            "    sample_rate: 16000\n",
            "    batch_size: 8\n",
            "    shuffle: false\n",
            "    num_workers: 8\n",
            "    pin_memory: true\n",
            "    use_start_end_token: true\n",
            "    \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NeMo I 2023-02-28 17:41:09 features:286] PADDING: 0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[NeMo W 2023-02-28 17:41:10 nemo_logging:349] /usr/local/lib/python3.8/dist-packages/torch/nn/modules/rnn.py:67: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
            "      warnings.warn(\"dropout option adds dropout after all but last \"\n",
            "    \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NeMo I 2023-02-28 17:41:11 rnnt_models:206] Using RNNT Loss : warprnnt_numba\n",
            "    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.0}\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:numba.cuda.cudadrv.driver:Call to cuInit results in CUDA_ERROR_NO_DEVICE\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[NeMo I 2023-02-28 17:41:12 save_restore_connector:247] Model EncDecRNNTBPEModel was successfully restored from /root/.cache/torch/NeMo/NeMo_1.16.0rc0/stt_de_conformer_transducer_large/183f232a57704e78fd9b211a3e021f02/stt_de_conformer_transducer_large.nemo.\n"
          ]
        }
      ],
      "source": [
        "import nemo.collections.asr as nemo_asr\n",
        "asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(model_name=\"stt_de_conformer_transducer_large\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HYkx10eEurz5"
      },
      "source": [
        "*********************************************************"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "# This is where the an4/ directory will be placed.\n",
        "# Change this if you don't want the data to be extracted in the current directory.\n",
        "data_dir = '.'\n",
        "\n",
        "if not os.path.exists(data_dir):\n",
        "  os.makedirs(data_dir)"
      ],
      "metadata": {
        "id": "IA5LmvRyuTSj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/ASR-Accent-Analysis-De.zip -d /content"
      ],
      "metadata": {
        "id": "8BLtPwGb1BFL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b69My7nmRF9q"
      },
      "outputs": [],
      "source": [
        "!pip install hyphenate\n",
        "!pip install python-Levenshtein"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ptgMGEOuRF9q"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pickle\n",
        "import pandas as pd\n",
        "import json\n",
        "import re\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from IPython.display import Audio\n",
        "from IPython.display import display\n",
        "from scipy.io import wavfile\n",
        "from itertools import chain \n",
        "import string\n",
        "import Levenshtein as Lev\n",
        "from itertools import groupby\n",
        "import scipy.stats as st\n",
        "from scipy import signal\n",
        "import nltk\n",
        "from scipy.stats import wasserstein_distance as wd\n",
        "from sklearn.metrics.pairwise import euclidean_distances as ed\n",
        "from pyemd import emd\n",
        "from hyphenate import hyphenate_word\n",
        "from itertools import islice \n",
        "from collections import OrderedDict"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oM6vhd0CRF9r"
      },
      "source": [
        "###<font color=\"00ff00\">  **Set up dataframe for transcripts and files:**</font>\n",
        "<font color=white> **Set up dataframe for transcripts and files- Test_accent.txt file case:**</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RM-H-0eTRF9r"
      },
      "outputs": [],
      "source": [
        "# Import the dataset file by method1 \n",
        "# from google.colab import files\n",
        "# uploaded = files.upload()\n",
        "if not os.path.exists('/content/Data/'):\n",
        "  os.makedirs('/content/Data/')\n",
        "! cp /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/cv-corpus04072022/de/validated.tsv /content/Data/\n",
        "# copy the expermintations files to deal with them\n",
        "! cp /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/ASR-Accent-Analysis-De/Data/*.* /content/Data/\n",
        "# copy the expermintations files from Mozilla Commen Voice v 10 to deal with them\n",
        "\n",
        "# import shutil\n",
        "# shutil.rmtree('/content/audio', ignore_errors=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WycLrfavlSVR"
      },
      "outputs": [],
      "source": [
        "#@title read in all three CSV files at once and store them in separate dataframes\n",
        "\n",
        "\n",
        "# # my_data/all_overlap.txt ==>> test_at.txt\n",
        "# # transcript ==>> text\n",
        "# # file==>> audio_filepath\n",
        "# # all_overlap ==>> dataset_trans_test_all_duration_test_all_output.csv\n",
        "# ###############################################\n",
        "\n",
        "# df_at, df_gb, df_it, df_de_al, df_fr, df_de_ni, df_ch, df_de, df_us, df_ca, df_ru = pd.read_csv(['test_at.txt', 'test_gb.txt', 'test_it.txt', 'test_de_al.txt', 'test_fr.txt',\n",
        "#                                'test_de_ni.txt', 'test_ch.txt', 'test_de.txt', 'test_us.txt', 'test_ca.txt', 'test_ru.txt'] )\n",
        "\n",
        "MCV_all=pd.DataFrame({})\n",
        "\n",
        "\n",
        "df_at= pd.read_csv('/content/Data/test_at.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_gb= pd.read_csv('/content/Data/test_gb.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_it= pd.read_csv( '/content/Data/test_it.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_de_al= pd.read_csv('/content/Data/test_de_al.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_fr= pd.read_csv('/content/Data/test_fr.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_de_ni= pd.read_csv( '/content/Data/test_de_ni.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_ch= pd.read_csv('/content/Data/test_ch.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_de= pd.read_csv('/content/Data/test_de.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_us= pd.read_csv( '/content/Data/test_us.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_ca= pd.read_csv('/content/Data/test_ca.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        "df_ru= pd.read_csv('/content/Data/test_ru.txt',delimiter = \", \", header = None ,encoding=\"utf-8\", names=['audio_filepath','transcript','duration'],engine='python' )\n",
        " \n",
        "# combine the dataframes into a single one \n",
        "combined_df = pd.concat([df_at, df_gb, df_it, df_de_al, df_fr, df_de_ni, df_ch, df_de, df_us, df_ca, df_ru], ignore_index=True)\n",
        "\n",
        "dataset_trans_test_all_duration=combined_df\n",
        "\n",
        "dataset_trans_test_all_duration['audio_filepath'] = dataset_trans_test_all_duration['audio_filepath'].map(lambda x: x.split('.',6)[2])\n",
        "dataset_trans_test_all_duration['audio_filepath'] = dataset_trans_test_all_duration['audio_filepath'].map(lambda x: x.split('/',6)[3])\n",
        "dataset_trans_test_all_duration['transcript'] = dataset_trans_test_all_duration['transcript'].map(lambda x: x.split('\": \"',2)[1])\n",
        "dataset_trans_test_all_duration['transcript'] = dataset_trans_test_all_duration['transcript'].map(lambda x: x.split('\"',2)[0])\n",
        "dataset_trans_test_all_duration['duration'] = dataset_trans_test_all_duration['duration'].map(lambda x: x.split('\": ',2)[1])\n",
        "dataset_trans_test_all_duration['duration'] = dataset_trans_test_all_duration['duration'].map(lambda x: x.split('}',2)[0])\n",
        "    \n",
        "\n",
        "print(type(dataset_trans_test_all_duration),'\\n',dataset_trans_test_all_duration.head(10))\n",
        "\n",
        "print(len(dataset_trans_test_all_duration))\n",
        "# the complete final dataframe of all Accents txt files\n",
        "dataset_trans_test_all_duration.to_csv( 'dataset_trans_test_all_duration.csv')\n",
        "\n",
        "MCV_all['audio_filepath']=dataset_trans_test_all_duration['audio_filepath']\n",
        "MCV_all['transcript']=dataset_trans_test_all_duration['transcript']\n",
        "\n",
        "MCV_all.to_csv( '/content/Data/MCV_all.csv')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "30lC5H_wLoWM"
      },
      "outputs": [],
      "source": [
        "MCV_all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kf26Pu1rRF9t"
      },
      "outputs": [],
      "source": [
        "dataset_trans_test_all_duration['transcript']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "544K28TzRF9t"
      },
      "outputs": [],
      "source": [
        "transcripts = list(set(dataset_trans_test_all_duration['transcript'].tolist()))\n",
        "len(transcripts)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6vdGmOHcRF9t"
      },
      "outputs": [],
      "source": [
        "type(transcripts)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dk3HX6crRF9u"
      },
      "outputs": [],
      "source": [
        "trans_dict = {x:[] for x in transcripts}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yCTPjzySRF9u"
      },
      "outputs": [],
      "source": [
        "len(trans_dict)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u031LcfARF9u"
      },
      "outputs": [],
      "source": [
        "# Assign the row['transcript'] value to the \n",
        "for index, row in dataset_trans_test_all_duration.iterrows():\n",
        "    trans_dict[row['transcript']].append(row['audio_filepath'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zMKZdPcxRF9u"
      },
      "outputs": [],
      "source": [
        "trans_dict['schreib ihr halt ein paar liebe worte rein']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dzjvieZjRF9u"
      },
      "outputs": [],
      "source": [
        "######################## show the first 5 rows of the dictionary trans_dict ##################################\n",
        "from itertools import islice\n",
        "\n",
        "def take(n, iterable):\n",
        "    \"\"\"Return the first n items of the iterable as a list.\"\"\"\n",
        "    return list(islice(iterable, n))\n",
        "\n",
        "n_items = take(5, trans_dict.items())\n",
        "n_items \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E_0TjQHmRF9u"
      },
      "outputs": [],
      "source": [
        "##### find the Audio file name\n",
        "trans_dict['findest du dich allein in braunschweig zurecht']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UswOlVb1HV5N"
      },
      "outputs": [],
      "source": [
        "##### find the Audio file name\n",
        "trans_dict['schreib ihr halt ein paar liebe worte rein']\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AoCIoeeNRF9v"
      },
      "source": [
        "###<font color=\"00ff00\">  **Set up dataframe for transcripts and files:**</font>\n",
        "<font color=white> **Set up dataframe for transcripts and files- Results.json file case:**</font> \n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UtkFSfiMRF9v"
      },
      "outputs": [],
      "source": [
        "#@title store the contents of Accented files in an extra text files respekt Ä Ö Ü ß\n",
        "################################## store the contents of Accented files in an extra text files respekt Ä Ö Ü ß ##################################### \n",
        "####################################################################################################################################################\n",
        "import json\n",
        "if __name__ == \"__main__\":\n",
        "  results = json.load(open('/content/Data/results.json'),encoding=\"utf-8\")\n",
        "  # Required_text=input(\"inser thte sentence: \")\n",
        "  for test_file in results:\n",
        "    list_values = [v[\"reference\"] for v in results[test_file].values()]\n",
        "    # string(ref_lens)\n",
        "    # print(f'{test_file} \\n',ref_lens)\n",
        "    str_values = ', '.join(str(x) for x in list_values)\n",
        "    \n",
        "    # print(df_trans.head(10))\n",
        "\n",
        "    # len(df_trans)\n",
        "\n",
        "    import os\n",
        "    # remove the script if exists \n",
        "    # os.remove(\"dataset_accent.py\")\n",
        "    # open script to write in the calculation of Mean of all accent\n",
        "    dataset_accent_write_file = open(f'/content/Data/dataset_{test_file}.py','w')\n",
        "    dataset_accent_write_file.write(str_values)\n",
        "    dataset_accent_write_file.close()\n",
        "    ! cp /content/*.txt.py /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/data/test_data/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yjz8UWrlRF9x"
      },
      "outputs": [],
      "source": [
        "#@title Creat Dictionary \"trans_dict_results\" of all Accent Audio files number and their Values\n",
        "\n",
        "########################################################################################################################################################\n",
        "############################# Creat Dictionary \"trans_dict_results\" of all Accent Audio files number and their Values###################################\n",
        "########################################################################################################################################################\n",
        "\n",
        "\n",
        "trans_dict_results = {'Keys_MMM_2050': '1000'}\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  Dict_results = json.load(open('/content/Data/results.json'))\n",
        "  for test_file in Dict_results:\n",
        "    list_test_accent_txt_values = [v[\"reference\"] for v in Dict_results[test_file].values()]\n",
        "    # print('*********\\n',list_test_accent_txt_values)\n",
        "    # list_test_accent_txt_keys=[k.split('.',6)[2] for k in Dict_results[test_file].keys()]\n",
        "    list_test_accent_txt_keys=[re.split(r'[.|/]',k)[9] for k in Dict_results[test_file].keys()]\n",
        "\n",
        "    trans_dict_test_file_result = dict(zip(list_test_accent_txt_keys, list_test_accent_txt_values))\n",
        "    trans_dict_results.update(trans_dict_test_file_result)\n",
        "    \n",
        "######################################## write the dictionary \"trans_dict\" to a json file ###############################\n",
        "with open(\"trans_dict_results.json\",\"w\", encoding='utf-8') as jsonfile:  \n",
        "  json.dump(trans_dict_results,jsonfile,ensure_ascii=False)\n",
        "! cp /content/trans_dict_results.json /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/data/test_data/\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KJjrdk_pRF9x"
      },
      "outputs": [],
      "source": [
        "#@title return first 5 items od dict trans_dict_results\n",
        "### Show the first 10 elements of the dict\n",
        "n_items = take(5, trans_dict_results.items())\n",
        "n_items "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "znI-C6vVRF9x"
      },
      "source": [
        "### <font color=white> **Display the Audio files number(Dict Key) of a given Utterance (Dict Value) and vice versa:**</font> "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fqDUrkNPRF9x"
      },
      "source": [
        "### <font color=\"00ff00\">  **Display the Audio files number(Dict Key) of a given Word (Dict Value):**</font>\n",
        "<font color=white> **We selected the fllowing 8 Words rather than 8 Utterances, due no enough Utterances in all Accents:**</font> \n",
        "\n",
        "<font color=white>\n",
        "\n",
        "1. höchsten \n",
        "2. völlig \n",
        "3. südlichen\n",
        "4. gefährlich\n",
        "5. hauptsächlich\n",
        "6. später\n",
        "7. geschäftsordnung\n",
        "8. präsidentschaft \n",
        "</font> \n",
        "\n",
        "\n",
        "### <font color=white> ****</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h6zhgwsDRF9y"
      },
      "outputs": [],
      "source": [
        "# ###############################################################\n",
        "# #### show the Audio files for a given Word ####################\n",
        "# for key, value in trans_dict_results.items():\n",
        "#       if \"höchsten\" in value:\n",
        "#         print(key)\n",
        "#         Audio_file_key=key"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HpTxYVutT7KS"
      },
      "outputs": [],
      "source": [
        "#@title ###**According to a given utterance, this code shows and writes in which Audio files and which Accent this given utterance is located:**\n",
        "\n",
        "###########################################################################################################################################################\n",
        "########### creat a Dataframe called \"dataset_audiofilename_transcript_accent\" contains all audiofilename and thier 'transcript' and 'test_file' /Accent####\n",
        "###########################################################################################################################################################\n",
        "list_dataset_test_=['test_at.txt','test_gb.txt','test_it.txt','test_de_al.txt','test_fr.txt','test_de_ni.txt','test_ch.txt',\n",
        "               'test_de.txt','test_us.txt','test_ca.txt','test_ru.txt']\n",
        "list_dataset_test_accent=['Österreichisches Deutsch','Britisches Deutsch','Italienisch Deutsch','Alemannische Färbung,Schweizer Standart Deutsch',\n",
        "'Französisch Deutsch','Niederländisch Deutsch','Schweizerdeutsch','Deutschland Deutsch','Amerikanisches Deutsch','Kanadisches Deutsch','Russisch Deutsch']\n",
        "\n",
        "data_dict_accent_duration  = dict(zip( list_dataset_test_, list_dataset_test_accent))  \n",
        "\n",
        "\n",
        "### here is the required_utterance\n",
        "required_utterance=\"findest du dich allein in braunschweig zurecht\"\n",
        "newfile=0\n",
        "# dict_audiofilename_transcript_accent=[]\n",
        "# To find out the Audio file's Accent\n",
        "list_test_accent_txt_values=[]\n",
        "list_test_accent_txt_keys=[]\n",
        "list_test_file=[]\n",
        "list_accent_long=[]\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  Dict_results = json.load(open('/content/Data/results.json'))\n",
        "  for test_file in Dict_results:\n",
        "    # print(test_file)\n",
        "    list_test_accent_txt_values_temp = [v[\"reference\"] for v in Dict_results[test_file].values()]\n",
        "    list_test_accent_txt_keys_temp=[re.split(r'[.|/]',k)[9] for k in Dict_results[test_file].keys()]\n",
        "    list_test_file_temp=[test_file for v in Dict_results[test_file].values()]\n",
        "\n",
        "# creat a Dataframe called  dataset_audiofilename_transcript_accent contains all audiofilename and thier 'transcript' and 'test_file' /Accent\n",
        "    list_test_accent_txt_values.extend(list_test_accent_txt_values_temp)\n",
        "    list_test_accent_txt_keys.extend(list_test_accent_txt_keys_temp)\n",
        "    list_test_file.extend(list_test_file_temp)\n",
        "    keys=['audiofilename','transcript','test_file']\n",
        "    trans_dict_test_file_result = dict(zip(keys,[list_test_accent_txt_keys, list_test_accent_txt_values,list_test_file]))\n",
        "    \n",
        "dataset_audiofilename_transcript_accent = pd.DataFrame(trans_dict_test_file_result)\n",
        "\n",
        "#############################################################################################\n",
        "##### Show and write which Audio files and which Accent the given word is located ###########\n",
        "\n",
        "for key, value in dataset_audiofilename_transcript_accent['transcript'].items():\n",
        "  if required_utterance in value:\n",
        "\n",
        "# to show the full name accent of the founded result \n",
        "    for keys_accent_long_i, values_accent_long_i in data_dict_accent_duration.items():\n",
        "      if dataset_audiofilename_transcript_accent.test_file[key]==keys_accent_long_i:\n",
        "        print()\n",
        "        # print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "        # f'*** Accent ={dataset_audiofilename_transcript_accent.test_file[key]}')\n",
        "        print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "              f'*** Accent is {data_dict_accent_duration[keys_accent_long_i]}')\n",
        "\n",
        "\n",
        "        # # Alternative way (for speed performance) to export the Audio files numbers and thiers Acccent to an external file\n",
        "        # # os.mkdir('/content/audiofilenames_transcript')\n",
        "\n",
        "        # if newfile==0:\n",
        "        #   dataset_accent_write_file = open(f'audiofilenames_transcript_accent_of_an_utterance.txt','w')\n",
        "        #   newfile=1\n",
        "        # else:\n",
        "        #   dataset_accent_write_file = open(f'audiofilenames_transcript_accent_of_an_utterance.txt','a')\n",
        "        #   dataset_accent_write_file.write(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "        #       f'*** Accent is {data_dict_accent_duration[keys_accent_long_i]}\\n')\n",
        "        #   dataset_accent_write_file.close()\n",
        "        #   # ! cp /content/audiofilenames_transcript/*.txt /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/data/test_data/audiofilenames_transcript/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YnGo6Bk6Bk6I"
      },
      "outputs": [],
      "source": [
        "#@title ###**According to a given word, this code shows and writes in which Audio files and which Accent this given word is located:**\n",
        "\n",
        "###########################################################################################################################################################\n",
        "########### creat a Dataframe called \"dataset_audiofilename_transcript_accent\" contains all audiofilename and thier 'transcript' and 'test_file' /Accent####\n",
        "###########################################################################################################################################################\n",
        "list_dataset_test_=['test_at.txt','test_gb.txt','test_it.txt','test_de_al.txt','test_fr.txt','test_de_ni.txt','test_ch.txt',\n",
        "               'test_de.txt','test_us.txt','test_ca.txt','test_ru.txt']\n",
        "list_dataset_test_accent=['Österreichisches Deutsch','Britisches Deutsch','Italienisch Deutsch','Alemannische Färbung,Schweizer Standart Deutsch',\n",
        "'Französisch Deutsch','Niederländisch Deutsch','Schweizerdeutsch','Deutschland Deutsch','Amerikanisches Deutsch','Kanadisches Deutsch','Russisch Deutsch']\n",
        "\n",
        "data_dict_accent_duration  = dict(zip( list_dataset_test_, list_dataset_test_accent))  \n",
        "\n",
        "\n",
        "### here is the required_word\n",
        "required_word=\"höchsten\"\n",
        "newfile=0\n",
        "# dict_audiofilename_transcript_accent=[]\n",
        "# To find out the Audio file's Accent\n",
        "list_test_accent_txt_values=[]\n",
        "list_test_accent_txt_keys=[]\n",
        "list_test_file=[]\n",
        "list_accent_long=[]\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  Dict_results = json.load(open('/content/Data/results.json'))\n",
        "  for test_file in Dict_results:\n",
        "    # print(test_file)\n",
        "    list_test_accent_txt_values_temp = [v[\"reference\"] for v in Dict_results[test_file].values()]\n",
        "    list_test_accent_txt_keys_temp=[re.split(r'[.|/]',k)[9] for k in Dict_results[test_file].keys()]\n",
        "    list_test_file_temp=[test_file for v in Dict_results[test_file].values()]\n",
        "\n",
        "# creat a Dataframe called  dataset_audiofilename_transcript_accent contains all audiofilename and thier 'transcript' and 'test_file' /Accent\n",
        "    list_test_accent_txt_values.extend(list_test_accent_txt_values_temp)\n",
        "    list_test_accent_txt_keys.extend(list_test_accent_txt_keys_temp)\n",
        "    list_test_file.extend(list_test_file_temp)\n",
        "    keys=['audiofilename','transcript','test_file']\n",
        "    trans_dict_test_file_result = dict(zip(keys,[list_test_accent_txt_keys, list_test_accent_txt_values,list_test_file]))\n",
        "    \n",
        "dataset_audiofilename_transcript_accent = pd.DataFrame(trans_dict_test_file_result)\n",
        "\n",
        "#############################################################################################\n",
        "##### Show and write which Audio files and which Accent the given word is located ###########\n",
        "\n",
        "for key, value in dataset_audiofilename_transcript_accent['transcript'].items():\n",
        "  if required_word in value:\n",
        "\n",
        "# to show the full name accent of the founded result \n",
        "    for keys_accent_long_i, values_accent_long_i in data_dict_accent_duration.items():\n",
        "      if dataset_audiofilename_transcript_accent.test_file[key]==keys_accent_long_i:\n",
        "        print()\n",
        "        # print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "        # f'*** Accent ={dataset_audiofilename_transcript_accent.test_file[key]}')\n",
        "        print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "              f'*** Accent is {data_dict_accent_duration[keys_accent_long_i]}')\n",
        " \n",
        "\n",
        "          ## Alternative way (for speed performance) to export the Audio files numbers and thiers Acccent to an external file\n",
        "        if newfile==0:\n",
        "          dataset_accent_write_file = open(f'Audiofilesnumber_Accent.txt','w')\n",
        "          newfile=1\n",
        "        else:\n",
        "          dataset_accent_write_file = open(f'Audiofilesnumber_Accent_{required_word}.txt','a')\n",
        "          # dataset_accent_write_file.write(f'{key}the Audio file\\'s Accent file is {test_file}\\n')\n",
        "          dataset_accent_write_file.write(f' {dataset_audiofilename_transcript_accent.audiofilename[key]} ***{dataset_audiofilename_transcript_accent.transcript[key]} *** Accent is {data_dict_accent_duration[keys_accent_long_i]}')\n",
        "\n",
        "\n",
        "          dataset_accent_write_file.close()\n",
        "          ! cp /content/Audiofilesnumber_Accent*.txt /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/data/test_data/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nLsbbysR-KmQ"
      },
      "outputs": [],
      "source": [
        "for key, value in trans_dict_test_file_result.items():\n",
        "    print(key, value)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EAB_n7YFBrAW"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "#@title ###**According to a given word, this code shows and writes in which Audio files and which Accent this given word is located:**\n",
        "\n",
        "###########################################################################################################################################################\n",
        "########### creat a Dataframe called \"dataset_audiofilename_transcript_accent\" contains all audiofilename and thier 'transcript' and 'test_file' /Accent####\n",
        "###########################################################################################################################################################\n",
        "list_dataset_test_=['test_at.txt','test_gb.txt','test_it.txt','test_de_al.txt','test_fr.txt','test_de_ni.txt','test_ch.txt',\n",
        "               'test_de.txt','test_us.txt','test_ca.txt','test_ru.txt']\n",
        "list_dataset_test_accent=['Österreichisches Deutsch','Britisches Deutsch','Italienisch Deutsch','Alemannische Färbung,Schweizer Standart Deutsch',\n",
        "'Französisch Deutsch','Niederländisch Deutsch','Schweizerdeutsch','Deutschland Deutsch','Amerikanisches Deutsch','Kanadisches Deutsch','Russisch Deutsch']\n",
        "\n",
        "data_dict_accent_duration  = dict(zip( list_dataset_test_, list_dataset_test_accent))  \n",
        "\n",
        "# create an empty set to store the words that meet all 11 accents\n",
        "words = set()\n",
        "accent_long_set=set()\n",
        "accent_long_list=[]\n",
        " \n",
        "\n",
        "### here is the required_word\n",
        "required_word=\"Start\"\n",
        "# required_word=input('Insert a word to test it please? ')\n",
        "newfile=0\n",
        "\n",
        "# dict_audiofilename_transcript_accent=[]\n",
        "# To find out the Audio file's Accent\n",
        "list_test_accent_txt_values=[]\n",
        "list_test_accent_txt_keys=[]\n",
        "list_test_file=[]\n",
        "list_accent_long=[]\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  Dict_results = json.load(open('/content/Data/results.json'))\n",
        "  for test_file in Dict_results:\n",
        "    # print(test_file)\n",
        "    list_test_accent_txt_values_temp = [v[\"reference\"] for v in Dict_results[test_file].values()]\n",
        "    list_test_accent_txt_keys_temp=[re.split(r'[.|/]',k)[9] for k in Dict_results[test_file].keys()]\n",
        "    list_test_file_temp=[test_file for v in Dict_results[test_file].values()]\n",
        "\n",
        "# creat a Dataframe called  dataset_audiofilename_transcript_accent contains all audiofilename and thier 'transcript' and 'test_file' /Accent\n",
        "    list_test_accent_txt_values.extend(list_test_accent_txt_values_temp)\n",
        "    list_test_accent_txt_keys.extend(list_test_accent_txt_keys_temp)\n",
        "    list_test_file.extend(list_test_file_temp)\n",
        "    keys=['audiofilename','transcript','test_file']\n",
        "    trans_dict_test_file_result = dict(zip(keys,[list_test_accent_txt_keys, list_test_accent_txt_values,list_test_file]))\n",
        "    \n",
        "dataset_audiofilename_transcript_accent = pd.DataFrame(trans_dict_test_file_result)\n",
        " \n",
        "\n",
        "for key, row in dataset_audiofilename_transcript_accent.iterrows():\n",
        "  if required_word in row['transcript']:\n",
        "\n",
        "# to show the full name accent of the founded result \n",
        "    for keys_accent_long_i, values_accent_long_i in data_dict_accent_duration.items():\n",
        "      if dataset_audiofilename_transcript_accent.test_file[key]==keys_accent_long_i:\n",
        "        # print()\n",
        "        # print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "        # f'*** Accent ={dataset_audiofilename_transcript_accent.test_file[key]}')\n",
        "        # print(dataset_audiofilename_transcript_accent.audiofilename[key],dataset_audiofilename_transcript_accent.transcript[key],\n",
        "        #       f'*** Accent is {data_dict_accent_duration[keys_accent_long_i]}')\n",
        "        accent_long_list.append(data_dict_accent_duration[keys_accent_long_i])\n",
        "        accent_long_str = ', '.join(accent_long_list)\n",
        "        accent_long_set=set(accent_long_str.split(', '))\n",
        "        # print(accent_long_str)\n",
        "\n",
        "        # get the accents for the current row\n",
        "        # accents = set(row['test_file'].split())\n",
        "        # accents = set(accent_long_set.split())\n",
        "        # print(len(accents))\n",
        "        # print('*'*60)\n",
        "        # check if the set of accents contains all 11 accents\n",
        "\n",
        "if len(accent_long_set) == 11:\n",
        "  print('*'*60)\n",
        "  print(f'Perfect, the Word {required_word} are found in all German accents')\n",
        "  print('*'*60)\n",
        "else:\n",
        "  print(f'unfortunately, the Word \"{required_word}\" are ***NOT*** found in all German accents')\n",
        "        # add the word to the set of words that meet all 11 accents\n",
        "#             words.add(row['transcript'])\n",
        "\n",
        "# # # print the words that meet all 11 accents\n",
        "# print('*'*60)\n",
        "# print(words)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8_xKKOkkpDKA"
      },
      "outputs": [],
      "source": [
        "dataset_audiofilename_transcript_accent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mbDXpv9E04en"
      },
      "source": [
        "###<font color=\"00ff00\">  **To count the number of occurrences of each word over the sentences located in the transcripts column of a data frame:**</font>\n",
        "<font color=white> **the following steps are achieve that**</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KIRokGFa01HZ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "# First, import the required libraries. You will need the pandas library to work with data frames and the nltk library to tokenize the sentences and words in the transcripts column.\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "nltk.download('punkt')\n",
        "\n",
        "\n",
        "#@title ###**To count the number of occurrences of each word over the sentences located in the transcripts column of a data frame:**\n",
        "\n",
        "###########################################################################################################################################################\n",
        "########### creat a Dataframe called \"dataset_audiofilename_transcript_accent\" contains all audiofilename and thier 'transcript' and 'test_file' /Accent####\n",
        "###########################################################################################################################################################\n",
        "list_dataset_test_=['test_at.txt','test_gb.txt','test_it.txt','test_de_al.txt','test_fr.txt','test_de_ni.txt','test_ch.txt',\n",
        "               'test_de.txt','test_us.txt','test_ca.txt','test_ru.txt']\n",
        "list_dataset_test_accent=['Österreichisches Deutsch','Britisches Deutsch','Italienisch Deutsch','Alemannische Färbung,Schweizer Standart Deutsch',\n",
        "'Französisch Deutsch','Niederländisch Deutsch','Schweizerdeutsch','Deutschland Deutsch','Amerikanisches Deutsch','Kanadisches Deutsch','Russisch Deutsch']\n",
        "\n",
        "data_dict_accent_duration  = dict(zip( list_dataset_test_, list_dataset_test_accent))  \n",
        "\n",
        "# create an empty set to store the words that meet all 11 accents\n",
        "\n",
        "\n",
        "accent_long_list=[]\n",
        " \n",
        "\n",
        "### here is the required_word\n",
        "# required_word=\"Start\"\n",
        "# required_word=input('Insert a word to test it please? ')\n",
        "newfile=0\n",
        "\n",
        "\n",
        "# dict_audiofilename_transcript_accent=[]\n",
        "# To find out the Audio file's Accent\n",
        "list_test_accent_txt_values=[]\n",
        "list_test_accent_txt_keys=[]\n",
        "list_test_file=[]\n",
        "list_accent_long=[]\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  Dict_results = json.load(open('/content/Data/results.json'))\n",
        "  for test_file in Dict_results:\n",
        "    # print(test_file)\n",
        "    list_test_accent_txt_values_temp = [v[\"reference\"] for v in Dict_results[test_file].values()]\n",
        "    list_test_accent_txt_keys_temp=[re.split(r'[.|/]',k)[9] for k in Dict_results[test_file].keys()]\n",
        "    list_test_file_temp=[test_file for v in Dict_results[test_file].values()]\n",
        "\n",
        "# creat a Dataframe called  dataset_audiofilename_transcript_accent contains all audiofilename and thier 'transcript' and 'test_file' /Accent\n",
        "    list_test_accent_txt_values.extend(list_test_accent_txt_values_temp)\n",
        "    list_test_accent_txt_keys.extend(list_test_accent_txt_keys_temp)\n",
        "    list_test_file.extend(list_test_file_temp)\n",
        "    keys=['audiofilename','transcript','test_file']\n",
        "    trans_dict_test_file_result = dict(zip(keys,[list_test_accent_txt_keys, list_test_accent_txt_values,list_test_file]))\n",
        "    \n",
        "dataset_audiofilename_transcript_accent = pd.DataFrame(trans_dict_test_file_result)\n",
        " \n",
        "# Tokenize the sentences in the transcripts column using the sent_tokenize function from the nltk library. This will create a list of sentences.\n",
        "sentences = dataset_audiofilename_transcript_accent['transcript'].apply(sent_tokenize)\n",
        "\n",
        "# Tokenize the words in each sentence using the word_tokenize function from the nltk library. This will create a list of words for each sentence.\n",
        "words = sentences.apply(lambda x: [word_tokenize(sentence) for sentence in x])\n",
        "\n",
        "# Flatten the list of words so that you have a single list of all words in the transcripts column.\n",
        "all_words = [word for sentence in words for word_list in sentence for word in word_list]\n",
        "\n",
        "# Use the Counter function from the collections library to count the occurrences of each word.\n",
        "from collections import Counter\n",
        "word_counts = Counter(all_words)\n",
        "\n",
        "# Convert the word_counts object to a data frame using the pd.DataFrame function.\n",
        "word_counts_dataset_audiofilename_transcript_accent = pd.DataFrame.from_dict(word_counts, orient='index', columns=['count'])\n",
        "\n",
        "# Sort the data frame by the count column in descending order.\n",
        "word_counts_dataset_audiofilename_transcript_accent = word_counts_dataset_audiofilename_transcript_accent.sort_values('count', ascending=False)\n",
        "\n",
        "print('Count the number of occurrences of each word over the sentences located in the transcripts column of a data frame:')\n",
        "print(word_counts_dataset_audiofilename_transcript_accent)\n",
        "word_counts_dataset_audiofilename_transcript_accent.to_csv('number_of_occurrences_of_each_word.csv', index=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sbKCg5nFwDaV"
      },
      "outputs": [],
      "source": [
        "# #@title Default title text\n",
        "\n",
        "# #@title Waveform_of_Audio_Examples\n",
        "# %matplotlib inline\n",
        "# import librosa.display\n",
        "# import matplotlib.pyplot as plt\n",
        "# import librosa\n",
        "# import glob\n",
        "# from IPython.display import Audio\n",
        "# import IPython.display as ipd\n",
        "\n",
        "\n",
        "\n",
        "# #@title display Audio (Wav) files\n",
        "# import librosa\n",
        "# import glob\n",
        "# from IPython.display import Audio\n",
        "# import IPython.display as ipd\n",
        "\n",
        "# i=0\n",
        "\n",
        "# # Set the path to the folder containing the WAV files\n",
        "# wav_path = \"/content/audio_wav_files_De\"\n",
        "\n",
        "# # Use glob to get a list of all WAV files in the folder\n",
        "# wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "\n",
        "\n",
        "# # Loop through the list of WAV files and play each file\n",
        "# for example_file in wav_files:\n",
        "#   # print(example_file)\n",
        "#     #Load the audio file using librosa\n",
        "#     audio, sample_rate = librosa.load(example_file)\n",
        "\n",
        "#     # Create an IPython.display.Audio object and play the audio\n",
        "#     # Load and listen to the audio file\n",
        "#     audio_display = Audio(audio, rate=sample_rate)\n",
        "#     display(audio_display)\n",
        "#     # ipd.Audio(example_file, rate=sample_rate)\n",
        "\n",
        "#     files = [os.path.join(data_dir, example_file)]\n",
        "#     for fname, transcription in zip(files, asr_model.transcribe(paths2audio_files=files)):\n",
        "#       print(f\"Audio in {fname} was recognized as: {transcription}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4-ea6GHH32bw"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MxBKH0H31-p"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qkwp7YLbxgq4"
      },
      "source": [
        "## <font color=\"00ff00\">  **3- Gradient-based Analysis**</font>\n",
        "<font color=white> **simple gradient-based explanation method considers the gradient of the output $f_j$ from a neural network (where j denotes a target class) with respect to an input $x_i$ (where i refers to the ith input time-step used to index the input sequence $x$):**</font> \n",
        "\n",
        "<font color=white>\n",
        "$grad(j,i,x)={\\displaystyle \\frac{\\partial f_j}{\\partial x_i}}$</font>\n",
        "\n",
        "\n",
        " <font color=\"00ff00\">  **3.1 Attribution Analysis**</font>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sRefw7qIRF9y"
      },
      "source": [
        "###<font color=\"00ff00\">  **Normalize Attributions**</font>\n",
        "<font color=white> **Normalize Attributions**</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ymf0aTLCjyZZ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "tsv_data = pd.read_csv('/content/Data/validated.tsv', sep='\\t')\n",
        "tsv_data.drop(tsv_data[(tsv_data['accents'].isna())].index, inplace=True)\n",
        "# tsv_data\n",
        "# print('*******************print Labels************************')\n",
        "\n",
        "# for label, content in tsv_data.items():\n",
        "#     print(f'label: {label}')\n",
        "#     #print(f'content: {content}', sep='\\n')\n",
        "\n",
        "# print('*********************print Contents**********************')\n",
        "\n",
        "# for label, content in tsv_data.items():\n",
        "#     #print(f'label: {label}')\n",
        "#     print(f'content: {content}', sep='\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xs1nmf6EBw0C"
      },
      "outputs": [],
      "source": [
        "index = tsv_data[tsv_data['path'] == 'common_voice_de_18946081.mp3'].index[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lNZS5u6sB0ti"
      },
      "outputs": [],
      "source": [
        "index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yqstDnuxB-FY"
      },
      "outputs": [],
      "source": [
        "tsv_data['accents'][index]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EX4eoB_RcMWl"
      },
      "outputs": [],
      "source": [
        "#@title Creat a dictionary for the Audio files , Accents\n",
        " \n",
        "import csv\n",
        "\n",
        "with open('/content/Data/results-AOE.json', 'r', encoding=\"utf-8\") as j:\n",
        "\tfile_meta_json = json.load(j)\n",
        " \n",
        "\n",
        "# Create an empty dictionary\n",
        "file_meta = {}\n",
        "\n",
        "# Loop through each key-value pair in the JSON data\n",
        "for accent_txt, audio_data in file_meta_json.items():\n",
        "\t# Loop through each audio file in the audio data\n",
        "\t# print(audio_data)\n",
        "\tfor audio_file, info in audio_data.items():\n",
        "\t\t# print(audio_file)\t\n",
        "\t\tinfo['audio']=re.split(r'[/|/|.]',audio_file)[9] \n",
        "\n",
        "\t\t# print(info)\n",
        "\t\t# Get the transcript from the reference field\n",
        "\t\ttranscript = info['reference']\n",
        "\n",
        "\t\t# delete the txt extension from the accent\n",
        "\t\taccent=accent_txt.split('.',2)[0]\n",
        "\t\t# Create a new inner dictionary with the accent and transcript\n",
        "\t\tinner_dict = {'accent': accent, 'transcript': transcript}\n",
        "\t\t# # Add the inner dictionary to the file_meta dictionary\n",
        "\t\tfile_meta[info['audio']] = inner_dict\n",
        "\n",
        "\n",
        "\n",
        " \n",
        "# Open a new CSV file in write mode\n",
        "with open('/content/Data/file_meta.csv', 'w', newline='') as file:\n",
        "    writer = csv.writer(file)\n",
        "\n",
        "    # Write the header row\n",
        "    writer.writerow(file_meta.keys())\n",
        "\n",
        "    # Write the values row\n",
        "    writer.writerow(file_meta.values())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vijs3C8AcMWm"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18568843']['accent']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mcpUldQMcMWm"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18568843']['transcript']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R1SbqtjfcMWn"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18568843'].keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U_y_0x_ZcMWn"
      },
      "outputs": [],
      "source": [
        "######################## show the first 5 rows of the dictionary trans_dict ##################################\n",
        "from itertools import islice\n",
        "\n",
        "def take(n, iterable):\n",
        "    \"\"\"Return the first n items of the iterable as a list.\"\"\"\n",
        "    return list(islice(iterable, n))\n",
        "\n",
        "n_items = take(5, file_meta.items())\n",
        "n_items "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i8tfT7eERF9z"
      },
      "source": [
        "###<font color=\"00ff00\">  **Preparing the data Audio files mp3, wav:**</font>\n",
        "<font color=white> **Convert the data Audio MP3 files to WAV files :**</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3A9wLc2VRGpC"
      },
      "outputs": [],
      "source": [
        "import shutil\n",
        "shutil.rmtree('/content/audio', ignore_errors=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JMAzadIAjJR0"
      },
      "outputs": [],
      "source": [
        "! pip install pydub\n",
        "from pydub import AudioSegment\n",
        "import os\n",
        "import glob\n",
        "from pathlib import Path"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "voJ8oMpyZ7eh"
      },
      "outputs": [],
      "source": [
        "#@title convert the mp3 data Audio files to Wav Audio files and store them rep. their accents\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# creat the folder audio/mp3/audio_mp3_files_De\n",
        "if not os.path.exists('audio/mp3/audio_mp3_files_De'):\n",
        "    os.makedirs('audio/mp3/audio_mp3_files_De')\n",
        "# copy the Asuio mp3 files to it\n",
        "! cp  /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/ASR-Accent-Analysis-De/audio_mp3_files_De/*.* /content/audio/mp3/audio_mp3_files_De\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# Set the path to the folder containing the MP3 files\n",
        "mp3_path = \"/content/audio/mp3/audio_mp3_files_De/*.mp3\"\n",
        "\n",
        "# Use glob to get a list of all MP3 files in the folder\n",
        "mp3_files = glob.glob(mp3_path)\n",
        "\n",
        "# Create an empty list to store the file names\n",
        "file_names = []\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# Loop through the list of MP3 files\n",
        "for mp3_file in mp3_files:\n",
        "\n",
        "  mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "  accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "  accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "  # Set the path to the folder where the converted WAV files will be saved\n",
        "  # Moreover add the accent name at the end of the folder \n",
        "  wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "  # Create the folder to store the WAV files, if it doesn't exist\n",
        "  if not os.path.exists(wav_path):\n",
        "    os.makedirs(wav_path)\n",
        "\n",
        "  # print(mp3_file)\n",
        "  # Load the MP3 file using pydub\n",
        "  audio = AudioSegment.from_mp3(mp3_file)\n",
        "\n",
        "  # Set the path and filename for the output WAV file\n",
        "  wav_file = os.path.join(wav_path, os.path.splitext(os.path.basename(mp3_file))[0] + \".wav\")\n",
        "\n",
        "  # Export the audio to WAV format\n",
        "  audio.export(wav_file, format=\"wav\")\n",
        "  print(f'Successfully Convert mp3 files to wav files of the Accent \"{accent_temp}\"')    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n_2HbebLSyJS"
      },
      "outputs": [],
      "source": [
        "! wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_de_conformer_transducer_large/versions/1.6.0/zip -O stt_de_conformer_transducer_large_1.6.0.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S1LSo0T0TKwG"
      },
      "outputs": [],
      "source": [
        "# #@title display Audio (Wav) files\n",
        "# import librosa\n",
        "# import glob\n",
        "# from IPython.display import Audio\n",
        "# import IPython.display as ipd\n",
        "\n",
        "\n",
        "# # Set the path to the folder containing the MP3 files\n",
        "# mp3_path = \"/content/audio/mp3/audio_mp3_files_De/*.mp3\"\n",
        "\n",
        "# # Use glob to get a list of all MP3 files in the folder\n",
        "# mp3_files = glob.glob(mp3_path)\n",
        "\n",
        "# # Create an empty list to store the file names\n",
        "# file_names = []\n",
        "\n",
        "\n",
        "# # Loop through the list of MP3 files\n",
        "# for mp3_file in mp3_files:\n",
        "\n",
        "#   mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "#   mp3_file_extension=re.split(r'[/|/]',mp3_file)[5] \n",
        "#   accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "#   accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "#   # Set the path to the folder where the converted WAV files will be saved\n",
        "#   # Moreover add the accent name at the end of the folder \n",
        "#   wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "#   # Use glob to get a list of all WAV files in the folder\n",
        "#   wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "#   # Loop through the list of WAV files and play each file\n",
        "#   for example_file in wav_files:\n",
        "#     # Load the audio file using librosa\n",
        "#     audio, sample_rate = librosa.load(example_file)\n",
        "\n",
        "#     # Create an IPython.display.Audio object and play the audio\n",
        "#     # Load and listen to the audio file\n",
        "#     audio_display = Audio(audio, rate=sample_rate)\n",
        "#     display(audio_display)\n",
        "#     # ipd.Audio(example_file, rate=sample_rate)\n",
        "    \n",
        "#     # print(f\"The Accent of the Aduio file {mp3_file_id} is {file_meta[mp3_file_id]['accent']}\")\n",
        "#     index_var = tsv_data[tsv_data['path'] == mp3_file_extension].index[0]\n",
        "#     print(f\"The Accent of the Aduio file {mp3_file_id} is {tsv_data['accents'][index_var]}\")\n",
        "#     print('*'*60)\n",
        "#     print('transcript of the Audio file: \"',file_meta[f'{mp3_file_id}']['transcript'],'\"','\\n')\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "files = [os.path.join(data_dir, '/content/audio/wav/audio_wav_files_De_test_ru/common_voice_de_21223629.wav')]\n",
        "for fname, transcription in zip(files, asr_model.transcribe(paths2audio_files=files)):\n",
        "  print(f\"Audio in {fname} was recognized as: {transcription}\")"
      ],
      "metadata": {
        "id": "LGxn09pwvN4Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jPvb-B_aD1F1"
      },
      "outputs": [],
      "source": [
        "#@title Display the Audio files and Recognize the German Accents\n",
        "\n",
        "%matplotlib inline\n",
        "import librosa.display\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa\n",
        "import glob\n",
        "from IPython.display import Audio\n",
        "import IPython.display as ipd\n",
        "\n",
        "i=0\n",
        "\n",
        "# Loop through the list of MP3 files\n",
        "for mp3_file in mp3_files:\n",
        "\n",
        "  mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "  mp3_file_extension=re.split(r'[/|/]',mp3_file)[5] \n",
        "  accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "  accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "  # Set the path to the folder where the converted WAV files will be saved\n",
        "  # Moreover add the accent name at the end of the folder \n",
        "  wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "  # Use glob to get a list of all WAV files in the folder\n",
        "  wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "  # Loop through the list of WAV files and play each file\n",
        "  for example_file_path in wav_files:\n",
        "    # Load the audio file using librosa\n",
        "    audio, sample_rate = librosa.load(example_file_path)\n",
        "    example_file=re.split(r'[/|/|.]',example_file_path)[5]\n",
        "    # Create an IPython.display.Audio object and play the audio\n",
        "    # Load and listen to the audio file\n",
        "    audio_display = Audio(audio, rate=sample_rate)\n",
        "    display(audio_display)\n",
        "    # ipd.Audio(example_file, rate=sample_rate)\n",
        "\n",
        "    files = [os.path.join(data_dir, example_file_path)]\n",
        "    for fname, transcription in zip(files, asr_model.transcribe(paths2audio_files=files)):\n",
        "      print(f\"Aduio transcript in {fname} recognized as: \\n{transcription}\",'\\n')\n",
        "      index_var = tsv_data[tsv_data['path'] == mp3_file_extension].index[0]\n",
        "      print(f\"The Accent of the Aduio file {example_file} is {tsv_data['accents'][index_var]}\")\n",
        "      print('--'*60)\n",
        "      print('transcript of the Audio file: \"',file_meta[f'{example_file}']['transcript'],'\"','\\n') \n",
        "      print('--'*60)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HTLSD52ccPqw"
      },
      "outputs": [],
      "source": [
        "audio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A-hpufoAbJcc"
      },
      "outputs": [],
      "source": [
        "trans_dict['abbildungen mit einer lipschitzkonstante kleiner als eins nennt man kontraktionen']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oQIAsGG9bV8D"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_17298952']['transcript']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ucbtxvGATTw7"
      },
      "outputs": [],
      "source": [
        "# #@title Waveform_of_Audio_Examples\n",
        "# %matplotlib inline\n",
        "# import librosa.display\n",
        "# import matplotlib.pyplot as plt\n",
        "# import librosa\n",
        "# import glob\n",
        "# from IPython.display import Audio\n",
        "# import IPython.display as ipd\n",
        "\n",
        "\n",
        "# i=0\n",
        "\n",
        "# # Set the path to the folder containing the WAV files\n",
        "# # Loop through the list of MP3 files\n",
        "# for mp3_file in mp3_files:\n",
        "\n",
        "#   mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "#   mp3_file_extension=re.split(r'[/|/]',mp3_file)[5] \n",
        "#   accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "#   accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "#   # Set the path to the folder where the converted WAV files will be saved\n",
        "#   # Moreover add the accent name at the end of the folder \n",
        "#   wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "#   # Use glob to get a list of all WAV files in the folder\n",
        "#   wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "#   # Loop through the list of WAV files and play each file\n",
        "#   for example_file in wav_files:\n",
        "#     # Load the audio file using librosa\n",
        "#     audio, sample_rate = librosa.load(example_file)\n",
        "\n",
        "\n",
        "#     # Plot the mel spectrogram of our sample in figure of 12 sectors to fit 11 German Accent\n",
        "#     figure_Waveform_of_Audio_Example=plt.figure(12,figsize=(25,25))  \n",
        "\n",
        "#     if i <= 6:\n",
        "#       ax_Waveform_of_Audio_Example_123=figure_Waveform_of_Audio_Example.add_subplot(3,4,i+1)\n",
        "#       i+=1\n",
        "#     else:\n",
        "#       break\n",
        "#     # Plot our example audio file's waveform\n",
        "#     plt.rcParams['figure.figsize'] = (15,7)\n",
        "#     plt.title('Waveform of Audio Example')\n",
        "#     plt.ylabel('Amplitude')\n",
        "#     _ = librosa.display.waveshow(audio)\n",
        "# plt.savefig('Waveform_of_Audio_Examples.png',facecolor='white')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H8As0vR0TZIE"
      },
      "outputs": [],
      "source": [
        "#@title Audio Spectrogram\n",
        "import numpy as np \n",
        "\n",
        "i=0\n",
        "\n",
        "# Loop through the list of MP3 files\n",
        "for mp3_file in mp3_files:\n",
        "\n",
        "  mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "  mp3_file_extension=re.split(r'[/|/]',mp3_file)[5] \n",
        "  accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "  accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "  # Set the path to the folder where the converted WAV files will be saved\n",
        "  # Moreover add the accent name at the end of the folder \n",
        "  wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "  # Use glob to get a list of all WAV files in the folder\n",
        "  wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "  # Loop through the list of WAV files and play each file\n",
        "  for example_file in wav_files:\n",
        "    # Load the audio file using librosa\n",
        "    audio, sample_rate = librosa.load(example_file)\n",
        "\n",
        "\n",
        "    # Plot the mel spectrogram of our sample in figure of 12 sectors to fit 11 German Accent\n",
        "    figure_Audio_Spectrogram=plt.figure(12,figsize=(25,25))  \n",
        "\n",
        "    if i <= 6:\n",
        "      ax_Audio_Spectrogram_123=figure_Audio_Spectrogram.add_subplot(3,4,i+1)\n",
        "      i+=1\n",
        "    else:\n",
        "      break\n",
        "\n",
        "\n",
        "    # Get spectrogram using Librosa's Short-Time Fourier Transform (stft)\n",
        "    spec = np.abs(librosa.stft(audio))\n",
        "    spec_db = librosa.amplitude_to_db(spec, ref=np.max)  # Decibels\n",
        "\n",
        "    # Use log scale to view frequencies\n",
        "    librosa.display.specshow(spec_db, y_axis='log', x_axis='time')\n",
        "    plt.colorbar()\n",
        "    plt.title('Audio Spectrogram');\n",
        "plt.savefig('Audio_Spectrogram.png',facecolor='white')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kNPEIPDjTZFC"
      },
      "outputs": [],
      "source": [
        "#@title Mel Spectrogram\n",
        "import numpy as np \n",
        "\n",
        "i=0\n",
        "\n",
        "# Loop through the list of MP3 files\n",
        "for mp3_file in mp3_files:\n",
        "\n",
        "  mp3_file_id=re.split(r'[/|/|.]',mp3_file)[5] \n",
        "  mp3_file_extension=re.split(r'[/|/]',mp3_file)[5] \n",
        "  accent_temp_txt=file_meta[mp3_file_id]['accent']\n",
        "  accent_temp=accent_temp_txt.split('.',2)[0]\n",
        "  # Set the path to the folder where the converted WAV files will be saved\n",
        "  # Moreover add the accent name at the end of the folder \n",
        "  wav_path = f'/content/audio/wav/audio_wav_files_De_{accent_temp}'\n",
        "\n",
        "  # Use glob to get a list of all WAV files in the folder\n",
        "  wav_files = glob.glob(wav_path + \"/*.wav\")\n",
        "  # Loop through the list of WAV files and play each file\n",
        "  for example_file in wav_files:\n",
        "    # Load the audio file using librosa\n",
        "    audio, sample_rate = librosa.load(example_file)\n",
        "\n",
        "    # Plot the mel spectrogram of our sample in figure of 12 sectors to fit 11 German Accent\n",
        "    figure_Audio_Spectrogram=plt.figure(12,figsize=(25,25))  \n",
        "    if i <= 6:\n",
        "      ax_Audio_Spectrogram_123=figure_Audio_Spectrogram.add_subplot(3,4,i+1)\n",
        "      i+=1\n",
        "    else:\n",
        "      break\n",
        "\n",
        "    # Plot the mel spectrogram of our sample\n",
        "    mel_spec = librosa.feature.melspectrogram(y=audio, sr=sample_rate)\n",
        "    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n",
        "\n",
        "    librosa.display.specshow(\n",
        "        mel_spec_db, x_axis='time', y_axis='mel')\n",
        "    plt.colorbar()\n",
        "    plt.title('Mel Spectrogram');\n",
        "\n",
        "plt.savefig('Mel_Spectrogram.png',facecolor='white')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fMfHndwcWmoC"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qu8eHbm8jX18"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18946081']['accent']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KGiS18AnrkJE"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18946081']['transcript']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lXoDG3hkVRo6"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18946081'].keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "raPbqMf_rvT0"
      },
      "outputs": [],
      "source": [
        "######################## show the first 5 rows of the dictionary trans_dict ##################################\n",
        "from itertools import islice\n",
        "\n",
        "def take(n, iterable):\n",
        "    \"\"\"Return the first n items of the iterable as a list.\"\"\"\n",
        "    return list(islice(iterable, n))\n",
        "\n",
        "n_items = take(5, file_meta.items())\n",
        "n_items "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dCil0nxzRF9z"
      },
      "outputs": [],
      "source": [
        "# Dipslay the transcript of an Audio file\n",
        "print(file_meta['common_voice_de_18568843']['accent'])\n",
        "print(dataset_trans_test_all_duration[dataset_trans_test_all_duration['audio_filepath'] == 'common_voice_de_18946081'].transcript.values)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LBqnV_cZ9IPf"
      },
      "outputs": [],
      "source": [
        "##### find the Audio file name\n",
        "trans_dict['schreib ihr halt ein paar liebe worte rein']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cjDqCbFzbJ3h"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18946081']"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.insert(0,'/content/ASR-Accent-Analysis-De/DeepSpeech')\n",
        "from data import data_loader"
      ],
      "metadata": {
        "id": "TojAMNT9_hfy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! python /content/ASR-Accent-Analysis-De/DeepSpeech/utils.py"
      ],
      "metadata": {
        "id": "nykEWUO3_j46"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title utils.py \n",
        "\n",
        "import torch\n",
        "import torch.distributed as dist\n",
        "\n",
        "from model_modified import DeepSpeech\n",
        "\n",
        "\n",
        "def reduce_tensor(tensor, world_size, reduce_op_max=False):\n",
        "    rt = tensor.clone()\n",
        "    dist.all_reduce(rt, op=dist.reduce_op.MAX if reduce_op_max is True else dist.reduce_op.SUM)  # Default to sum\n",
        "    if not reduce_op_max:\n",
        "        rt /= world_size\n",
        "    return rt\n",
        "\n",
        "\n",
        "def check_loss(loss, loss_value):\n",
        "    \"\"\"\n",
        "    Check that warp-ctc loss is valid and will not break training\n",
        "    :return: Return if loss is valid, and the error in case it is not\n",
        "    \"\"\"\n",
        "    loss_valid = True\n",
        "    error = ''\n",
        "    if loss_value == float(\"inf\") or loss_value == float(\"-inf\"):\n",
        "        loss_valid = False\n",
        "        error = \"WARNING: received an inf loss\"\n",
        "    elif torch.isnan(loss).sum() > 0:\n",
        "        loss_valid = False\n",
        "        error = 'WARNING: received a nan loss, setting loss value to 0'\n",
        "    elif loss_value < 0:\n",
        "        loss_valid = True\n",
        "        error = \"WARNING: received a negative loss\"\n",
        "    return loss_valid, error\n",
        "\n",
        "\n",
        "def load_model(device, model_path, use_half):\n",
        "    model = DeepSpeech.load_model(model_path)\n",
        "    model.eval()\n",
        "    model = model.to(device)\n",
        "    if use_half:\n",
        "        model = model.half()\n",
        "    return model"
      ],
      "metadata": {
        "id": "lq4yyjqv-zxB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title import lib packages to Colab \n",
        "# import sys\n",
        "# sys.path.append('/content/ASR-Accent-Analysis-De/deepspeech.pytorch/deepspeech_pytorch/loader')\n",
        "\n",
        "# # import all modules in the folder\n",
        "# from loader import spec_augment\n",
        "\n",
        "import sys\n",
        "sys.path.insert(0,'/content/ASR-Accent-Analysis-De/DeepSpeech')\n",
        "from data import data_loader\n",
        "\n",
        "\n",
        "\n",
        "# import sys\n",
        "# sys.path.insert(0,'/content/ASR-Accent-Analysis-De/deepspeech.pytorch/deepspeech_pytorch/loader')\n",
        "# import sparse_image_warp "
      ],
      "metadata": {
        "id": "20LAxd0i2wMU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title 111Changed/Modified files of DeepSpeech\n",
        "! python /content/ASR-Accent-Analysis-De/DeepSpeech/test_attr.py --model-path /content/ASR-Accent-Analysis-De/STT_De_Conformer_Transducer_Large/model/stt_de_conformer_transducer_large.nemo --test-manifest /content/ASR-Accent-Analysis-De/Data/dataset_trans_test_all_duration.csv --cuda  "
      ],
      "metadata": {
        "id": "vlP_hXIz0yoO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import wave\n",
        "import pickle\n",
        "# os.\n",
        "# Step 2: Read the audio file\n",
        "with wave.open('/content/audio/wav/audio_wav_files_De_test_de/common_voice_de_18568843.wav', 'rb') as audio_file:\n",
        "    sample_rate = audio_file.getframerate()\n",
        "    num_channels = audio_file.getnchannels()\n",
        "    audio_data = audio_file.readframes(-1)\n",
        "\n",
        "# Step 3: Create a dictionary\n",
        "data_dict = {\n",
        "    'audio_data': audio_data,\n",
        "    'sample_rate': sample_rate,\n",
        "    'num_channels': num_channels\n",
        "}\n",
        "\n",
        "# Step 4: Save the dictionary as a pickle file\n",
        "with open('common_voice_de_18568843.pkl', 'wb') as f:\n",
        "    pickle.dump(data_dict, f)"
      ],
      "metadata": {
        "id": "iCbQg6ZR1pq6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.makedirs('attribution/grad')"
      ],
      "metadata": {
        "id": "0TQxdAFT1pno"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "common_voice_de_18568843"
      ],
      "metadata": {
        "id": "E9LOlTGB1pkS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ijpo-CE6RF9z"
      },
      "outputs": [],
      "source": [
        "#@title get_norm_attr Function\n",
        "def get_norm_attr(index_dict):\n",
        "    norm_dict = {}\n",
        "    for key in index_dict.keys():\n",
        "        norm_dict[key] = index_dict[key]/np.sum(index_dict[key])\n",
        "        np.set_printoptions(precision=4)\n",
        "        #print(norm_dict[key])\n",
        "        #break\n",
        "    return norm_dict"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PmxhTbhKRF9z"
      },
      "source": [
        "###<font color=white> **Fetching frame allignmnets from the meta-data (using gentle)**</font> \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pP1ZlgF7RF90"
      },
      "outputs": [],
      "source": [
        "def get_frame_allignment(file, input_size):\n",
        "    alligned = []\n",
        "    spec_stride = 0.01\n",
        "    window_size = 0.02\n",
        "    times = file_meta[file]['end_times']\n",
        "    last_idx = 0\n",
        "    for i in range(input_size):\n",
        "        frame_idx = i\n",
        "        window_start = frame_idx*spec_stride\n",
        "        window_mid = window_start + (window_size/2)\n",
        "        alligned_phone = 'na'\n",
        "        for j in range(len(times)):\n",
        "            if (window_mid < times[j]):\n",
        "                alligned_phone = file_meta[file]['phones'][j]\n",
        "                break\n",
        "        #assert alligned_phone != 'na', \"Failed to fetch allignment\"\n",
        "        if(alligned_phone != 'na'):\n",
        "            alligned.append(alligned_phone)\n",
        "            last_idx = i\n",
        "    pause_start = 0\n",
        "    pause_end = len(alligned)\n",
        "    for i in range(len(alligned)):\n",
        "        if(alligned[i] != 'pause'):\n",
        "            break\n",
        "        pause_start = i\n",
        "    \n",
        "    for i in range(len(alligned)-1,-1,-1):\n",
        "        if(alligned[i] != 'pause'):\n",
        "            break\n",
        "        pause_end = i\n",
        "        \n",
        "    #print(last_idx)\n",
        "    #print(pause_start, pause_end)\n",
        "    return alligned, pause_start +1, pause_end\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uHOIVIllaoqs"
      },
      "outputs": [],
      "source": [
        "trans_dict['schreib ihr halt ein paar liebe worte rein']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HGNc_ZDXatsD"
      },
      "outputs": [],
      "source": [
        "file_meta['common_voice_de_18946081']['accent']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HrkJsWgWIKUe"
      },
      "outputs": [],
      "source": [
        "index_var = tsv_data[tsv_data['path'] == 'common_voice_de_18946081.mp3'].index[0]\n",
        "tsv_data['accents'][index_var]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file"
      ],
      "metadata": {
        "id": "IzYsOK-LE-mX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wbc2aISXRF90"
      },
      "source": [
        "###<font color=white> **Visualizing (signed) attributions**</font>  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H3RuE54JRF90"
      },
      "outputs": [],
      "source": [
        "# if __name__ == \"__main__\":\n",
        "#   Dict_results = json.load(open('results.json'))\n",
        "#   for accent in Dict_results:\n",
        "#     list_test_accent_txt_values = [v[\"reference\"] for v in Dict_results[accent].values()]\n",
        "\n",
        "for file in trans_dict['schreib ihr halt ein paar liebe worte rein']:\n",
        "    try:\n",
        "        #file = 'common_voice_de_18946081' for file in ['schreib ihr halt ein paar liebe worte rein']:\n",
        "        Fs, wav = wavfile.read('/content/audio/wav/audio_wav_files_De_{}/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "\n",
        "        # Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "        display(Audio(wav, rate=Fs))\n",
        "        with open('/content/attribution/grad/{}.pkl'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "        print(file_attr['output'], file_meta[file]['accent'])\n",
        "        print(list(file_attr['attr dict'].keys()))\n",
        "        normalized_attr = get_norm_attr(file_attr['attr dict'])\n",
        "        keys = list(normalized_attr.keys())\n",
        "        input_size = len(file_attr['attr dict'][list(file_attr['attr dict'].keys())[0]]) # calculate properly once\n",
        "        buffer = 10\n",
        "    #print(get_frame_allignment(file,input_size))\n",
        "    #     with np.printoptions(precision=4, suppress=True):\n",
        "    #         print(normalized_attr[1][:291])\n",
        "        plot_vertical = False\n",
        "        idx = 0\n",
        "        if(plot_vertical):\n",
        "            fig = plt.figure(figsize = (1,35))\n",
        "            print(get_frame_allignment(file, input_size))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "            #plt.yticks(rotation=) \n",
        "            #plt.yticks(range(allignments.shape[0]),list(allignments))\n",
        "            plt.show()\n",
        "        else:\n",
        "            \n",
        "            fig = plt.figure(figsize = (37,1))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            #actual_size = len(allignments)\n",
        "            \n",
        "            print('for',file_attr['output'][keys[idx]])\n",
        "            sns.heatmap(np.expand_dims(normalized_attr[keys[idx]][p_start - buffer:p_end + buffer], axis = 0), fmt=\"\", cmap='RdBu')\n",
        "            #plt.yticks(rotation=) \n",
        "            modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "            #phone_centres = []\n",
        "            phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "            len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "            idx_list = [0]\n",
        "            ticks_list = []\n",
        "            for l in len_list:\n",
        "                ticks_list.append(idx_list[-1] + l//2)\n",
        "                idx_list.append(idx_list[-1] + l)\n",
        "                \n",
        "           \n",
        "            plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "            \n",
        "            for j in idx_list:\n",
        "                plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "                plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "                \n",
        "                \n",
        "                \n",
        "            plt.show()\n",
        "    #break\n",
        "    except:\n",
        "        print('failed for file : {}'.format(file))\n",
        "        continue\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D_a-88MBRF90"
      },
      "source": [
        "###<font color=\"00ff00\">  **Visualizing (modulus/magnitude) attributions**</font>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xURhYJzBRF92"
      },
      "source": [
        "###<font color=white> Generate **|*Input.Gradient*|** Attributution at grapheme-level</font> "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_Kik3GLKRF92"
      },
      "outputs": [],
      "source": [
        "#for file in trans_dict[\"I'm going to them.\"]:\n",
        "def inp_grad_grapheme(file, idx):\n",
        "    try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "        with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "#         print(list(file_attr['attr dict'].keys()))\n",
        "        normalized_attr = get_norm_attr(file_attr['attr dict'])\n",
        "        keys = list(normalized_attr.keys())\n",
        "        input_size = len(file_attr['attr dict'][list(file_attr['attr dict'].keys())[0]]) # calculate properly once\n",
        "        buffer = 10\n",
        "    #print(get_frame_allignment(file,input_size))\n",
        "    #     with np.printoptions(precision=4, suppress=True):\n",
        "    #         print(normalized_attr[1][:291])\n",
        "        plot_vertical = False\n",
        "        #idx = 0\n",
        "        \n",
        "        if(plot_vertical):\n",
        "            fig = plt.figure(figsize = (1,35))\n",
        "            print(get_frame_allignment(file, input_size))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "            #plt.yticks(rotation=) \n",
        "            #plt.yticks(range(allignments.shape[0]),list(allignments))\n",
        "            plt.show()\n",
        "        else:\n",
        "            \n",
        "            fig = plt.figure(figsize = (37,1))\n",
        "            #print(allignments)\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            #actual_size = len(allignments)\n",
        "            \n",
        "            print('for',file_attr['output'][keys[idx]])\n",
        "            sns.heatmap(np.expand_dims(np.abs(normalized_attr[keys[idx]][p_start - buffer:p_end + buffer])/np.sum(np.abs(normalized_attr[keys[idx]][p_start - buffer:p_end + buffer])), axis = 0), fmt=\"\", cmap='Blues')\n",
        "            #plt.yticks(rotation=) \n",
        "            modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "            phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "            len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "            idx_list = [0]\n",
        "            ticks_list = []\n",
        "            for l in len_list:\n",
        "                ticks_list.append(idx_list[-1] + l//2)\n",
        "                idx_list.append(idx_list[-1] + l)\n",
        "                \n",
        "            #ticks_list.append((idx_list[-1] + len(modified_allignments))//2)   \n",
        "            \n",
        "            plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "            \n",
        "            for j in idx_list:\n",
        "                plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "                plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "            plt.show()\n",
        "                \n",
        "    #break\n",
        "    except:\n",
        "        print('failed for file : {}'.format(file))\n",
        "        #continue\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Dc1KGljRF92"
      },
      "source": [
        "##### Helper functions for word-level allignments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qsdRogv4RF93"
      },
      "outputs": [],
      "source": [
        "def get_space(inp):\n",
        "    spaces = []\n",
        "    for idx, val in enumerate(inp):\n",
        "        if(val == ' '): spaces.append(idx)\n",
        "    return spaces"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AKtlqHgPRF93"
      },
      "outputs": [],
      "source": [
        "def get_words(test_list, split_list,spaces):\n",
        "    temp = zip(chain([0], split_list), chain(split_list, [None])) \n",
        "    res = list(test_list[i : j] for i, j in temp) \n",
        "    #print(res)\n",
        "    final_res = []\n",
        "    for l in res:\n",
        "        if(l[0] in spaces):\n",
        "            final_res.append([l[0]])\n",
        "            final_res.append(l[1:])\n",
        "        else: \n",
        "            final_res.append(l)\n",
        "    #print(final_res)\n",
        "    return final_res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TzNBcp8BRF93"
      },
      "outputs": [],
      "source": [
        "def get_words_wo_space(test_list, split_list,spaces):\n",
        "    temp = zip(chain([0], split_list), chain(split_list, [None])) \n",
        "    res = list(test_list[i : j] for i, j in temp) \n",
        "    #print(res)\n",
        "    final_res = []\n",
        "    for l in res:\n",
        "        if(l[0] in spaces):\n",
        "            final_res.append([l[0]])\n",
        "            final_res.append(l[1:])\n",
        "        else: \n",
        "            final_res.append(l)\n",
        "    #print(final_res)\n",
        "    return final_res"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fIsVxQ9JRF93"
      },
      "source": [
        "##### Generate **|*Input.Gradient*|** Attributution at word-level "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y1T5YHOxRF93"
      },
      "outputs": [],
      "source": [
        "#for file in trans_dict[\"I'm going to them.\"]:\n",
        "def inp_grad_word(file, word_idx):\n",
        "    try:\n",
        "    #file = 'common_voice_en_110121'\n",
        "    #print(file)\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "        with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "\n",
        "    #         print(file_attr['output'].split(' '))\n",
        "        normalized_attr = get_norm_attr(file_attr['attr dict'])\n",
        "        keys = list(normalized_attr.keys())\n",
        "#         print(keys)\n",
        "        spaces = get_space(file_attr['output'])\n",
        "        indices = [keys.index(x) for x in spaces]\n",
        "        words = get_words(keys,indices, spaces)\n",
        "        input_size = len(file_attr['attr dict'][list(file_attr['attr dict'].keys())[0]]) # calculate properly once\n",
        "        buffer = 10\n",
        "        plot_vertical = False\n",
        "        #word_idx = 0\n",
        "\n",
        "        word_activation = np.sum(np.asarray([np.abs(normalized_attr[idx]) for idx in words[word_idx]] ), axis = 0)\n",
        "\n",
        "        if(plot_vertical):\n",
        "            fig = plt.figure(figsize = (1,35))\n",
        "            print(get_frame_allignment(file, input_size))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "\n",
        "            plt.show()\n",
        "        else:\n",
        "            str_list = []\n",
        "            for x in words[word_idx]:\n",
        "                str_list.append(file_attr['output'][x])\n",
        "\n",
        "\n",
        "            print(''.join(str_list))\n",
        "            fig = plt.figure(figsize = (37,1))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            #actual_size = len(allignments)\n",
        "\n",
        "            sns.heatmap(np.expand_dims(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer]), axis = 0), fmt=\"\", cmap='Blues')\n",
        "            #plt.yticks(rotation=) [p_start - buffer:p_end + buffer]\n",
        "            modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "            phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "            len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "            idx_list = [0]\n",
        "            ticks_list = []\n",
        "            for l in len_list:\n",
        "                ticks_list.append(idx_list[-1] + l//2)\n",
        "                idx_list.append(idx_list[-1] + l)\n",
        "\n",
        "\n",
        "            plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "\n",
        "            for j in idx_list:\n",
        "                plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "                plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "\n",
        "\n",
        "            plt.show()\n",
        "    \n",
        "    except:\n",
        "        print('failed for file : {}'.format(file))\n",
        "#         continue\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o2g1iHBnRF93"
      },
      "source": [
        "##### Helper functions to compare WERs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xaPkLxBcRF94"
      },
      "outputs": [],
      "source": [
        "def wer(s1, s2):\n",
        "        \"\"\"\n",
        "        Computes the Word Error Rate, defined as the edit distance between the\n",
        "        two provided sentences after tokenizing to words.\n",
        "        Arguments:\n",
        "            s1 (string): space-separated sentence\n",
        "            s2 (string): space-separated sentence\n",
        "        \"\"\"\n",
        "\n",
        "        # build mapping of words to integers\n",
        "        b = set(s1.split() + s2.split())\n",
        "        word2char = dict(zip(b, range(len(b))))\n",
        "\n",
        "        # map the words to a char array (Levenshtein packages only accepts\n",
        "        # strings)\n",
        "        w1 = [chr(word2char[w]) for w in s1.split()]\n",
        "        w2 = [chr(word2char[w]) for w in s2.split()]\n",
        "\n",
        "        return Lev.distance(''.join(w1), ''.join(w2))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CVwWCNSuRF94"
      },
      "outputs": [],
      "source": [
        "transcript_wer = {}\n",
        "#lm_wer = {}\n",
        "accent_lm_wer = {x: [] for x in ['us', 'indian','african','canada','australia','england','scotland']}\n",
        "accent_wer = {x: [] for x in ['us', 'indian','african','canada','australia','england','scotland']}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bHhc6kQrRF94"
      },
      "outputs": [],
      "source": [
        "with open('wers.pickle', 'rb') as l:\n",
        "    lm_wer = pickle.load(l)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YPMksaviRF94"
      },
      "outputs": [],
      "source": [
        "for transcript in transcripts:\n",
        "    transcript_ = transcript.strip().upper()\n",
        "    wer_dict = {}\n",
        "\n",
        "    valid_punctuation = string.punctuation.replace(\"'\",\"\")\n",
        "    processed_transcript = transcript_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    #print(processed_transcript)       \n",
        "    for file in trans_dict[transcript]:\n",
        "        try:\n",
        "            with open('attribution/{}.pickle'.format(file), 'rb') as j:\n",
        "                file_attr = pickle.load(j)\n",
        "            accent = file_meta[file]['accent']\n",
        "            \n",
        "            #print(accent)\n",
        "            output = file_attr['output'].replace('_', '')\n",
        "            #print(output)\n",
        "            num_tokens = len(processed_transcript.split())\n",
        "            wer_ = 100*wer(processed_transcript, output)/num_tokens\n",
        "            wer_dict[accent] = wer_\n",
        "            accent_wer[accent].append(wer_)\n",
        "            accent_lm_wer[accent].append(lm_wer[file]['wer'])\n",
        "            \n",
        "        except:\n",
        "            continue\n",
        "        \n",
        "    \n",
        "    transcript_wer[transcript] = wer_dict\n",
        "\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jVkC9f-tRF94"
      },
      "outputs": [],
      "source": [
        "print('accent','|', 'greedy','|', 'lm rescored','|', 'num of files')\n",
        "for key in accent_wer.keys():\n",
        "    print(key,'|', np.asarray(accent_wer[key]).mean(),'|', np.asarray(accent_lm_wer[key]).mean(),'|', len(accent_wer[key]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kpGBXs5ARF94"
      },
      "outputs": [],
      "source": [
        "#get_colour = {'us':'k', 'indian':'g','african':'b','canada':'r','australia':'c','england':'m','scotland':'y'}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LEAn4dp0RF94"
      },
      "source": [
        "##### Generate **|*Gradient*|** Attribution at grapheme-level"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fszXWxWzRF95"
      },
      "outputs": [],
      "source": [
        "#for file in trans_dict[\"I'm going to them.\"]:\n",
        "def grad_grapheme(file, idx):\n",
        "    try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "        with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "#         print(list(file_attr['grad_dict'].keys()))\n",
        "        normalized_attr = get_norm_attr(file_attr['grad_dict'])\n",
        "        keys = list(normalized_attr.keys())\n",
        "        input_size = len(file_attr['grad_dict'][list(file_attr['grad_dict'].keys())[0]]) # calculate properly once\n",
        "        buffer = 10\n",
        "    #print(get_frame_allignment(file,input_size))\n",
        "    #     with np.printoptions(precision=4, suppress=True):\n",
        "    #         print(normalized_attr[1][:291])\n",
        "        plot_vertical = False\n",
        "        #idx = 0\n",
        "\n",
        "        if(plot_vertical):\n",
        "            fig = plt.figure(figsize = (1,35))\n",
        "            print(get_frame_allignment(file, input_size))\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "            #plt.yticks(rotation=) \n",
        "            #plt.yticks(range(allignments.shape[0]),list(allignments))\n",
        "            plt.show()\n",
        "        else:\n",
        "\n",
        "            fig = plt.figure(figsize = (37,1))\n",
        "            #print(allignments)\n",
        "            allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "            allignments = np.asarray(allignments)            \n",
        "            actual_size = len(allignments)\n",
        "            #actual_size = len(allignments)\n",
        "\n",
        "            print('for',file_attr['output'][keys[idx]])\n",
        "            sns.heatmap(np.expand_dims(np.abs(normalized_attr[keys[idx]][p_start - buffer:p_end + buffer])/np.sum(np.abs(normalized_attr[keys[idx]][p_start - buffer:p_end + buffer])), axis = 0), fmt=\"\", cmap='Greens')\n",
        "            #plt.yticks(rotation=) \n",
        "            modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "            phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "            len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "            idx_list = [0]\n",
        "            ticks_list = []\n",
        "            for l in len_list:\n",
        "                ticks_list.append(idx_list[-1] + l//2)\n",
        "                idx_list.append(idx_list[-1] + l)\n",
        "\n",
        "            #ticks_list.append((idx_list[-1] + len(modified_allignments))//2)   \n",
        "\n",
        "            plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "\n",
        "            for j in idx_list:\n",
        "                plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "                plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "            plt.show()\n",
        "\n",
        "    #break\n",
        "    except:\n",
        "        print('failed for file : {}'.format(file))\n",
        "#         continue\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WdSG7nFlRF95"
      },
      "source": [
        "##### Generate **|*Gradient*|** Attribution at word-level"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MGTH_WKnRF95"
      },
      "outputs": [],
      "source": [
        "#for file in trans_dict[\"I'm going to them.\"]:\n",
        "def grad_word(file, word_idx):\n",
        "    #try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "        #print(file)\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "    with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "        file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "\n",
        "#         print(file_attr['output'].split(' '))\n",
        "    normalized_attr = get_norm_attr(file_attr['grad_dict'])\n",
        "    keys = list(normalized_attr.keys())\n",
        "#         print(keys)\n",
        "    spaces = get_space(file_attr['output'])\n",
        "    indices = [keys.index(x) for x in spaces]\n",
        "    words = get_words(keys,indices, spaces)\n",
        "    input_size = len(file_attr['grad_dict'][list(file_attr['grad_dict'].keys())[0]]) # calculate properly once\n",
        "    buffer = 10\n",
        "    plot_vertical = False\n",
        "    #word_idx = 8\n",
        "#         print(words)\n",
        "#         print(keys)\n",
        "\n",
        "    word_activation = np.sum(np.asarray([np.abs(normalized_attr[idx]) for idx in words[word_idx]] ), axis = 0)\n",
        "\n",
        "    if(plot_vertical):\n",
        "        fig = plt.figure(figsize = (1,35))\n",
        "        print(get_frame_allignment(file, input_size))\n",
        "        allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "        allignments = np.asarray(allignments)            \n",
        "        actual_size = len(allignments)\n",
        "        sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "\n",
        "        plt.show()\n",
        "    else:\n",
        "        str_list = []\n",
        "        for x in words[word_idx]:\n",
        "            str_list.append(file_attr['output'][x])\n",
        "\n",
        "\n",
        "        print('Focus word:',''.join(str_list))\n",
        "        fig = plt.figure(figsize = (37,1))\n",
        "        allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "        allignments = np.asarray(allignments)            \n",
        "        actual_size = len(allignments)\n",
        "        #actual_size = len(allignments)\n",
        "\n",
        "        sns.heatmap(np.expand_dims(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer]), axis = 0), fmt=\"\", cmap='Greens')\n",
        "        #plt.yticks(rotation=) [p_start - buffer:p_end + buffer]\n",
        "        modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "        phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "        len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "        idx_list = [0]\n",
        "        ticks_list = []\n",
        "        for l in len_list:\n",
        "            ticks_list.append(idx_list[-1] + l//2)\n",
        "            idx_list.append(idx_list[-1] + l)\n",
        "\n",
        "\n",
        "        plt.xticks(ticks_list,phone_labels, rotation = 90, fontsize=16)\n",
        "\n",
        "        for j in idx_list:\n",
        "            plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "            plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "\n",
        "\n",
        "        plt.show()\n",
        "#         fig2 = plt.figure(figsize = (37,1))\n",
        "#         my_arr = word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer])\n",
        "#         plt.plot(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer]))\n",
        "#         xk = np.arange(len(my_arr))\n",
        "#         #pk = (0.1, 0.2, 0.3, 0.1, 0.1, 0.0, 0.2)        \n",
        "#         custm = st.rv_discrete(name='custm', values=(xk, my_arr))\n",
        "#         print(custm.mean(), custm.std())\n",
        "#         #print(my_arr.mean(), my_arr.std())\n",
        "#         #plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "#         plt.xlim(xmin = 0, xmax = len(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer])))\n",
        "        \n",
        "        \n",
        "#         plt.show()\n",
        "\n",
        "#     except:\n",
        "#         print('failed for file : {}'.format(file))\n",
        "        #continue\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "elRd5eOxRF95"
      },
      "outputs": [],
      "source": [
        "target_transcript = \"I'm going to them.\" # indian gowmedo\n",
        "target_transcript = 'The burning fire had been extinguished.' #across accents\n",
        "#target_transcript = 'It was the first time she had done that.' #SHE across accents, sheld in usm hi in indian\n",
        "#target_transcript = 'I was scared, but wasted no time in going out and crossing the bridge to the sand pits.'\n",
        "grapheme_idx = 0\n",
        "word_idx = 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rL8ReQ-URF95"
      },
      "source": [
        "### Visualizing Attributions for all accents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__qKHbdVRF95"
      },
      "source": [
        "#### (A) Grapheme-level"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gr0LVpnhRF96"
      },
      "outputs": [],
      "source": [
        "fig = plt.figure(figsize = (37,20))\n",
        "for file in trans_dict[target_transcript]:\n",
        "    try:\n",
        "        Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "        display(Audio(wav, rate=Fs))\n",
        "        with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "        print(file_attr['output'], file_meta[file]['accent'])\n",
        "        #print(list(file_attr['attr dict'].keys()))\n",
        "        inp_grad_grapheme(file, grapheme_idx)\n",
        "        grad_grapheme(file,grapheme_idx)\n",
        "        \n",
        "    except:\n",
        "        continue"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lw8UId9_RF96"
      },
      "source": [
        "#### (B) Word Level"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RCE8Z5puRF96"
      },
      "outputs": [],
      "source": [
        "word_idx = 4\n",
        "for file in trans_dict[target_transcript]:\n",
        "    try:\n",
        "        Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "        display(Audio(wav, rate=Fs))\n",
        "        with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "            file_attr = pickle.load(j)\n",
        "        print(file_attr['output'], file_meta[file]['accent'])\n",
        "        #print(list(file_attr['attr dict'].keys()))\n",
        "        inp_grad_word(file, word_idx)\n",
        "        #print(file_meta[file]['accent'])\n",
        "        grad_word(file,word_idx)\n",
        "#     break\n",
        "    except:\n",
        "        continue"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "St6-4pxdRF96"
      },
      "outputs": [],
      "source": [
        "syllables = pickle.load(open('syll.pickle','rb'))\n",
        "syllables['THE']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WsK2V2JuRF96"
      },
      "outputs": [],
      "source": [
        "syllables['PEOPLE']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FTcYOPCxRF96"
      },
      "outputs": [],
      "source": [
        "def is_correct(s1, s2, idx):\n",
        "        \"\"\"\n",
        "        Computes the Word Error Rate, defined as the edit distance between the\n",
        "        two provided sentences after tokenizing to words.\n",
        "        Arguments:\n",
        "            s1 (string): space-separated sentence\n",
        "            s2 (string): space-separated sentence\n",
        "        \"\"\"\n",
        "\n",
        "        # build mapping of words to integers\n",
        "        b = set(s1.split() + s2.split())\n",
        "        word2char = dict(zip(b, range(len(b))))\n",
        "\n",
        "        # map the words to a char array (Levenshtein packages only accepts\n",
        "        # strings)\n",
        "        w1 = [chr(word2char[w]) for w in s1.split()]\n",
        "        w2 = [chr(word2char[w]) for w in s2.split()]\n",
        "        ops = Lev.editops(''.join(w1), ''.join(w2))\n",
        "        #print(ops)\n",
        "        words_changed = [x[1] for x in ops]\n",
        "        \n",
        "        return not idx in words_changed\n",
        "        \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JhyuYVloRF96"
      },
      "outputs": [],
      "source": [
        "cond_transcripts = []\n",
        "for t in transcripts:\n",
        "    \n",
        "    order = sorted(transcript_wer[t], key=lambda k: transcript_wer[t][k])\n",
        "    #print(order)\n",
        "    if(order[0]== 'canada' or order[0] == 'us'):\n",
        "        cond_transcripts.append(t)\n",
        "    \n",
        " # do this after LM rescoring ?   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Lg7vWBPRF98"
      },
      "outputs": [],
      "source": [
        "print(len(cond_transcripts))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mXvOV_3oRF99"
      },
      "outputs": [],
      "source": [
        "print(cond_transcripts[:20])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u1l2OnNyRF99"
      },
      "outputs": [],
      "source": [
        "all_words = []\n",
        "valid_punctuation = string.punctuation.replace(\"'\",\"\")\n",
        "for t in transcripts:\n",
        "    t_ = t.strip().upper()\n",
        "    t_ = t_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    all_words.extend(t_.split())\n",
        "    #print(all_words)\n",
        "    #break\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N2NXMdgZRF99"
      },
      "source": [
        "##### Calculate Most Frequent Words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c21DM02HRF99"
      },
      "outputs": [],
      "source": [
        "allWordDist = nltk.FreqDist(all_words)\n",
        "record_frequency = allWordDist.most_common(75)\n",
        "most_frequent = [ x[0] for x in record_frequency]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GTkE_V3NRF99"
      },
      "outputs": [],
      "source": [
        "print(most_frequent)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mF2v3QYBRF99"
      },
      "outputs": [],
      "source": [
        "most_frequent = ['THE', 'TO', 'AND', 'A', 'OF', 'WAS', 'I', 'IT', 'HE', 'THAT', 'IN', 'YOU', 'HAD', 'HIS', 'AS', 'BUT', 'WITH', 'BOY', 'IS', 'THEY', 'WERE', 'FOR', 'AT', 'ABOUT', 'BE', 'ON', 'ME', 'THERE', 'FROM', 'MY', 'WE', 'HIM', 'HAVE', 'NOT', 'OUT', 'THIS', 'SOME', 'ALL', 'THOUGHT', 'AN', 'PEOPLE', 'BEEN', 'HER', 'INTO', 'TIME', 'YOUR', 'SO', 'ARE', 'HERE', 'CAN', 'GET', 'THEN', 'WAY', 'SHE', 'ONE', 'WHEN', 'ONLY', \"DON'T\", \"I'M\", 'OTHER', 'UP', 'WHAT', 'SEE', 'COULD', 'LITTLE', 'NO', 'GOING', 'DO', 'WILL', 'IF', 'ITS', 'MORE', 'BY', 'MAN', 'STILL']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cd7KUNkRRF9-"
      },
      "outputs": [],
      "source": [
        "mf_us = {x:[] for x in most_frequent}\n",
        "mf_canada = {x:[] for x in most_frequent}\n",
        "mf_indian = {x:[] for x in most_frequent}\n",
        "mf_african = {x:[] for x in most_frequent}\n",
        "mf_england = {x:[] for x in most_frequent}\n",
        "mf_scotland = {x:[] for x in most_frequent}\n",
        "mf_australian = {x:[] for x in most_frequent}\n",
        "mf_accents = {'us':mf_us,'canada':mf_canada,'indian':mf_indian,'african':mf_african,'england':mf_england,'scotland':mf_scotland,'australia':mf_australian}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o1BYAhP4RF9-"
      },
      "outputs": [],
      "source": [
        "#transcript_freq = {}\n",
        "for t in transcripts:\n",
        "    \n",
        "    t_ = t.strip().upper()\n",
        "    t_ = t_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    \n",
        "    t_list = t_.split()\n",
        "    true_nz_counts = {x:t_list.count(x) for x in most_frequent if t_list.count(x) > 0}\n",
        "    true_nz_indices = {x:[index for index, value in enumerate(t_list) if value == x] for x in true_nz_counts.keys()}\n",
        "    #print(true_nz_counts)\n",
        "    for file in trans_dict[t]:\n",
        "        try:\n",
        "            acc = {x:0 for x in true_nz_counts.keys()}\n",
        "            with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "                file_attr = pickle.load(j)\n",
        "            op = file_attr['output'].replace('_', '')\n",
        "            accent = file_meta[file]['accent']\n",
        "            \n",
        "            \n",
        "            for w in true_nz_counts.keys():\n",
        "                \n",
        "                for j in true_nz_indices[w]:\n",
        "                #mf_accents[accent][w][0].append(true_nz_counts[w])\n",
        "                    \n",
        "                    mf_accents[accent][w].append(is_correct(t_,op,j))\n",
        "                #print(cond)\n",
        "           \n",
        "                #mf_accents[accent][w][1].append(min(op.split().count(w),true_nz_counts[w]))\n",
        "        except:\n",
        "            continue\n",
        "    \n",
        "        #print(mf_accents[accent])\n",
        "    \n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iQKrmqSORF9-"
      },
      "outputs": [],
      "source": [
        "#print(mf_us)\n",
        "mf_us = {x:[] for x in most_frequent}\n",
        "mf_canada = {x:[] for x in most_frequent}\n",
        "mf_indian = {x:[] for x in most_frequent}\n",
        "mf_african = {x:[] for x in most_frequent}\n",
        "mf_england = {x:[] for x in most_frequent}\n",
        "mf_scotland = {x:[] for x in most_frequent}\n",
        "mf_australian = {x:[] for x in most_frequent}\n",
        "avg_stats =  {'us':mf_us,'canada':mf_canada,'indian':mf_indian,'african':mf_african,'england':mf_england,'scotland':mf_scotland,'australia':mf_australian}\n",
        "#print(avg_stats)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YWY24rb4RF9-"
      },
      "outputs": [],
      "source": [
        "for a in mf_accents.keys():\n",
        "    print(a)\n",
        "    #a = 'indian'\n",
        "    for w in mf_accents[a].keys():\n",
        "        \n",
        "        avg_stats[a][w].append(sum(mf_accents[a][w])/len(mf_accents[a][w]))\n",
        "        \n",
        "        #print(w,np.asarray(mf_accents[a][w]).mean(),np.asarray(mf_accents[a][w]).std())\n",
        "    \n",
        "  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "23U8bzXVRF9-"
      },
      "source": [
        "##### Accuracy of correctly predicting most frequent words across accents"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uoMqceXQRF9-"
      },
      "outputs": [],
      "source": [
        "for a in avg_stats.keys():\n",
        "    temp = []\n",
        "    for k in avg_stats[a].keys():\n",
        "        temp.append(avg_stats[a][k][0])\n",
        "    print(a, np.asarray(temp).mean(), np.asarray(temp).std())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tYgEsSGjRF9_"
      },
      "outputs": [],
      "source": [
        "us = [0.7803950509449439, 0.121667382968697]\n",
        "canada =[0.8572825991328428, 0.10528083986888306]\n",
        "indian =[0.513035145387385, 0.18122269227720902]\n",
        "african =[0.7522708227207352, 0.1365991362807828]\n",
        "england =[0.6919775766172566, 0.16313311778908576]\n",
        "scotland =[0.6535836230311115, 0.1547649386757295]\n",
        "australia =[0.7459351700052215, 0.14893722495703424]\n",
        "acc = {'us':us,'indian':indian,'canada':canada,'african':african,'england':england,'scotland':scotland,'australia':australia}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oHPwfv4CRF9_"
      },
      "outputs": [],
      "source": [
        "objects = ['us', 'indian', 'african', 'canada', 'england', 'australia', 'scotland']\n",
        "y_pos = np.arange(len(objects))\n",
        "#fig = plt.figure(figsize = (15,10))\n",
        "# def create_plots(layer, name):\n",
        "fig = plt.figure(figsize = (10,5))\n",
        "y_pos = np.arange(len(objects))\n",
        "y = [100*acc[x][0] for x in objects ]\n",
        "err = [100*acc[x][1]  for  x in objects]\n",
        "#plt.plot(y_pos, y,'-o', alpha=0.7,)\n",
        "#plt.figure()\n",
        "plt.bar(y_pos, y, yerr= err,align='center', capsize=7, edgecolor='k')\n",
        "plt.xticks(y_pos, objects, fontsize=14)\n",
        "plt.yticks(fontsize=14)\n",
        "# plt.errorbar(y_pos, y, yerr=err,fmt='-o',capsize=5 )\n",
        "# plt.xticks(y_pos, objects)\n",
        "#plt.ylim(ymin = 0)obj\n",
        "#plt.axhline(14.28, linewidth=1, color='k')\n",
        "plt.ylabel('Accuracy %',color='k',fontsize=18)\n",
        "plt.xlabel('Accents',color='k',fontsize=18)\n",
        "#plt.title('MF words classificant trends')\n",
        "#plt.legend(frameon=False)\n",
        "plt.savefig('MF-words.pdf',bbox_inches='tight')\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_uEz4jv9RF9_"
      },
      "source": [
        "##### Helper Function for alligning words with frames"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dcAj1khiRF9_"
      },
      "outputs": [],
      "source": [
        "def get_word_allignment(file, input_size):\n",
        "    alligned = []\n",
        "    spec_stride = 0.01\n",
        "    window_size = 0.02\n",
        "    times = file_meta[file]['end_times']\n",
        "    json_path = 'my_data/align_common/{}.json'.format(file)\n",
        "    with open(json_path,'r') as j:\n",
        "        gentle = json.load(j)\n",
        "    word_ends = []\n",
        "    word_ends.append(('start',times[0]))\n",
        "    for g in range(len(gentle['words'])):\n",
        "        word_ends.append((gentle['words'][g]['word'], gentle['words'][g]['end']))\n",
        "    word_ends.append(('end',times[-1]))\n",
        "    \n",
        "    #last_idx = 0\n",
        "    for i in range(input_size):\n",
        "        frame_idx = i\n",
        "        window_start = frame_idx*spec_stride\n",
        "        window_mid = window_start + (window_size/2)\n",
        "        alligned_word = 'na'\n",
        "        for j in range(len(word_ends)):\n",
        "            if (window_mid < word_ends[j][1]):\n",
        "                alligned_word = word_ends[j][0]\n",
        "                break\n",
        "        #assert alligned_phone != 'na', \"Failed to fetch allignment\"\n",
        "        if(alligned_word != 'na'):\n",
        "            alligned.append(alligned_word)\n",
        "            #last_idx = i\n",
        "    pause_start = 0\n",
        "    pause_end = len(alligned)\n",
        "    for i in range(len(alligned)):\n",
        "        if(alligned[i] != 'start'):\n",
        "            break\n",
        "        pause_start = i\n",
        "    \n",
        "    for i in range(len(alligned)-1,-1,-1):\n",
        "        if(alligned[i] != 'end'):\n",
        "            break\n",
        "        pause_end = i\n",
        "        \n",
        "    #print(last_idx)\n",
        "    \n",
        "    return alligned, pause_start+1, pause_end,\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gJ2qU00yRF9_"
      },
      "outputs": [],
      "source": [
        "wrd = get_word_allignment('common_voice_en_540956', 100000)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8KR7XEv_RF-A"
      },
      "source": [
        "### Visualizing attribution of a particular word"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LdBEGLilRF-A"
      },
      "outputs": [],
      "source": [
        "def grad_clubbed_word(file, word_idx, disable_prints = False):\n",
        "    \n",
        "    #try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "        #print(file)\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "    with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "        file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "\n",
        "#         print(file_attr['output'].split(' '))\n",
        "    normalized_attr = get_norm_attr(file_attr['grad_dict'])\n",
        "    keys = list(normalized_attr.keys())\n",
        "#         print(keys)\n",
        "    spaces = get_space(file_attr['output'])\n",
        "    indices = [keys.index(x) for x in spaces]\n",
        "    words = get_words(keys,indices, spaces)\n",
        "    input_size = len(file_attr['grad_dict'][list(file_attr['grad_dict'].keys())[0]]) # calculate properly once\n",
        "    buffer = 10\n",
        "    plot_vertical = False\n",
        "    #word_idx = 8\n",
        "#         print(words)\n",
        "#         print(keys)\n",
        "\n",
        "    word_activation = np.sum(np.asarray([np.abs(normalized_attr[idx]) for idx in words[word_idx]] ), axis = 0)\n",
        "    \n",
        "\n",
        "    if(plot_vertical):\n",
        "        fig = plt.figure(figsize = (1,35))\n",
        "        print(get_frame_allignment(file, input_size))\n",
        "        allignments, p_start, p_end = get_word_allignment(file, input_size)\n",
        "        allignments = np.asarray(allignments)            \n",
        "        actual_size = len(allignments)\n",
        "        sns.heatmap(np.expand_dims(normalized_attr[1][:actual_size], axis = 1),annot = np.expand_dims(allignments, axis = 1), fmt=\"\", cmap='RdBu')\n",
        "\n",
        "        plt.show()\n",
        "    else:\n",
        "        str_list = []\n",
        "        for x in words[word_idx]:\n",
        "            str_list.append(file_attr['output'][x])\n",
        "\n",
        "\n",
        "        \n",
        "        allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "        wrd_allignments,w_start,w_end = get_word_allignment(file, input_size)\n",
        "        wrd_allignments = np.asarray(wrd_allignments)\n",
        "        allignments = np.asarray(allignments)            \n",
        "        actual_size = len(allignments)\n",
        "        #actual_size = len(allignments)\n",
        "        \n",
        "        #plt.yticks(rotation=) [p_start - buffer:p_end + buffer]\n",
        "        modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "        assert len(wrd_allignments) == len(allignments), \"dimensions don't match\"\n",
        "        \n",
        "        modified_wrd_allignments = wrd_allignments[w_start - buffer:w_end + buffer]\n",
        "        #assert len(modified_wrd_allignments) == len(modified_allignments), \"dimensions don't match\"\n",
        "        #wrds = [list(x[0]) for x in groupby(modified_wrd_allignments)]\n",
        "        wrd_labels = [list(x[1]) for x in groupby(modified_wrd_allignments)]\n",
        "        #print(wrd_labels)\n",
        "        wrd_indices = [0]\n",
        "        for j in wrd_labels:\n",
        "            wrd_indices.append(wrd_indices[-1] + len(j))\n",
        "        #print(wrd_indices)\n",
        "        my_arr = word_activation[w_start - buffer:w_end + buffer]/np.sum(word_activation[w_start - buffer:w_end + buffer])\n",
        "        wrd_combined_wonorm = []\n",
        "        wrd_combined = []\n",
        "        for m in range(len(wrd_labels)):\n",
        "            wrd_combined_wonorm.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]]))\n",
        "            wrd_combined.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]])/len(wrd_labels[m]))\n",
        "        wrd_combined = np.asarray(wrd_combined)\n",
        "        \n",
        "        wrd_combined = wrd_combined/np.sum(wrd_combined)\n",
        "        \n",
        "        #print(np.asarray(wrd_combined).sum())\n",
        "        if(not disable_prints):\n",
        "            print(''.join(str_list))\n",
        "            fig = plt.figure(figsize = (37,1))\n",
        "            sns.heatmap(np.expand_dims(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer]), axis = 0), fmt=\"\", cmap='Greens')\n",
        "            phone_labels = [x[0] for x in groupby(modified_allignments)]\n",
        "            len_list = [len(list(x[1])) for x in groupby(modified_allignments)]\n",
        "            idx_list = [0]\n",
        "            ticks_list = []\n",
        "            for l in len_list:\n",
        "                ticks_list.append(idx_list[-1] + l//2)\n",
        "                idx_list.append(idx_list[-1] + l)\n",
        "\n",
        "\n",
        "            plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "\n",
        "            for j in idx_list:\n",
        "                plt.axvline(x=j, color='w', linestyle='-', linewidth=2.5)\n",
        "                plt.axvline(x=j, color='k', linestyle='--', linewidth=2.5)\n",
        "\n",
        "\n",
        "            plt.show()\n",
        "            print(wrd_combined_wonorm)\n",
        "            print(wrd_combined)  \n",
        "            fig2 = plt.figure(figsize = (37,1))\n",
        "            #my_arr = word_activation[w_start - buffer:w_end + buffer]/np.sum(word_activation[w_start - buffer:w_end + buffer])\n",
        "            plt.plot(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer]))\n",
        "            xk = np.arange(len(my_arr))\n",
        "            #pk = (0.1, 0.2, 0.3, 0.1, 0.1, 0.0, 0.2)        \n",
        "            custm = st.rv_discrete(name='custm', values=(xk, my_arr))\n",
        "            print(custm.mean(), custm.std())\n",
        "              \n",
        "            #print(my_arr.mean(), my_arr.std())\n",
        "            #plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "            plt.xlim(xmin = 0, xmax = len(word_activation[p_start - buffer:p_end + buffer]/np.sum(word_activation[p_start - buffer:p_end + buffer])))\n",
        "\n",
        "\n",
        "            plt.show()\n",
        "\n",
        "            fig3 = plt.figure(figsize = (37,1))\n",
        "            my_arr = wrd_combined\n",
        "            plt.plot(my_arr)\n",
        "            xk = np.arange(len(my_arr))\n",
        "            #pk = (0.1, 0.2, 0.3, 0.1, 0.1, 0.0, 0.2)        \n",
        "            custm = st.rv_discrete(name='custm', values=(xk, my_arr))\n",
        "            print(custm.mean(), custm.std())\n",
        "            #print(my_arr.mean(), my_arr.std())\n",
        "            #plt.xticks(ticks_list,phone_labels, rotation = 90)\n",
        "            plt.xlim(xmin = 0, xmax = len(my_arr))\n",
        "\n",
        "\n",
        "            plt.show()\n",
        "        #else:\n",
        "        return wrd_combined, wrd_combined_wonorm\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-WlP5ZUxRF-A"
      },
      "source": [
        "##### Visualizing normalized attributions at granularity of frames and words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0P-BBkD2RF-A"
      },
      "outputs": [],
      "source": [
        "#target_transcript = 'I was scared, but wasted no time in going out and crossing the bridge to the sand pits.'\n",
        "\n",
        "for file in trans_dict[target_transcript]:\n",
        "    Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "    display(Audio(wav, rate=Fs))\n",
        "    with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "        file_attr = pickle.load(j)\n",
        "    print(file_attr['output'], file_meta[file]['accent'])\n",
        "    print(list(file_attr['attr dict'].keys()))\n",
        "    #inp_grad_grapheme(file, grapheme_idx)\n",
        "    print(file)\n",
        "    distr, distr_wo = grad_clubbed_word(file, 2*idx, disable_prints=False)\n",
        "                    #print(distr)\n",
        "    max_idx = np.argmax(np.asarray(distr))\n",
        "    max_idx_wo = np.argmax(np.asarray(distr_wo))\n",
        "    custm = st.rv_discrete(name='custm', values=(np.arange(len(distr)), distr))\n",
        "    spread = custm.expect(lambda x : (x- custm.mean())**2)\n",
        "    print(max_idx,spread**0.5)\n",
        "          \n",
        "    #break\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gOLfAZskRF-A"
      },
      "source": [
        "#### Analysis of attributions summed at word level (from transcription)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ocufOJ7YRF-A"
      },
      "outputs": [],
      "source": [
        "mf_us = {x:([],[],[]) for x in most_frequent}\n",
        "mf_canada = {x:([],[],[]) for x in most_frequent}\n",
        "mf_indian = {x:([],[],[]) for x in most_frequent}\n",
        "mf_african = {x:([],[],[]) for x in most_frequent}\n",
        "mf_england = {x:([],[],[]) for x in most_frequent}\n",
        "mf_scotland = {x:([],[],[]) for x in most_frequent}\n",
        "mf_australian = {x:([],[],[]) for x in most_frequent}\n",
        "max_accents = {'us':mf_us,'canada':mf_canada,'indian':mf_indian,'african':mf_african,'england':mf_england,'scotland':mf_scotland,'australia':mf_australian}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gm4D0JSBRF-B"
      },
      "outputs": [],
      "source": [
        "\n",
        "for t in transcripts:\n",
        "    \n",
        "    t_ = t.strip().upper()\n",
        "    t_ = t_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    #print(t_)\n",
        "    t_list = t_.split()\n",
        "    true_nz_counts = {x:t_list.count(x) for x in most_frequent if t_list.count(x) > 0}\n",
        "    true_nz_indices = {x:[index for index, value in enumerate(t_list) if value == x] for x in true_nz_counts.keys()}\n",
        "    \n",
        "    for file in trans_dict[t]:\n",
        "        \n",
        "        try:   \n",
        "            with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "                file_attr = pickle.load(j)\n",
        "            op = file_attr['output'].replace('_', '')\n",
        "            op_list = op.split()\n",
        "            accent = file_meta[file]['accent']\n",
        "\n",
        "            a_indices = {x:[index for index, value in enumerate(op_list) if value == x] for x in true_nz_counts.keys()}\n",
        "        \n",
        "            for w in true_nz_counts.keys():\n",
        "                for idx in a_indices[w]:\n",
        "                    # compute condition for correctness\n",
        "                    distr, distr_wo = grad_clubbed_word(file, 2*idx, disable_prints=True)\n",
        "                    #print(distr)\n",
        "                    max_idx = np.argmax(np.asarray(distr))\n",
        "                    max_idx_wo = np.argmax(np.asarray(distr_wo))\n",
        "                    \n",
        "            #print(custm.mean(), custm.std())\n",
        "                    if(is_correct(op,t_,idx)):\n",
        "                        custm = st.rv_discrete(name='custm', values=(np.arange(len(distr)), distr))\n",
        "                        spread = custm.expect(lambda x : (x - max_idx)**2)\n",
        "                        #print(spread)\n",
        "                        max_accents[accent][w][2].append(spread**0.5)\n",
        "                        \n",
        "                        idx2 = true_nz_indices[w][a_indices[w].index(idx)]\n",
        "                        max_accents[accent][w][0].append(max_idx == idx2 + 1)\n",
        "                        max_accents[accent][w][1].append(max_idx_wo == idx2 + 1)\n",
        "        except:\n",
        "            continue\n",
        "\n",
        "            \n",
        "        #print(mf_accents[accent])\n",
        "    \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-p9kv2rsRF-B"
      },
      "source": [
        "##### Accuracy of how often the word alligned from meta data has the highest cumulative attribtuion for words from transcription (given that word is transcribed correctly)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qvF7sx2GRF-B"
      },
      "outputs": [],
      "source": [
        "for a in max_accents.keys():\n",
        "    print(a)\n",
        "    acc = ([],[],[])\n",
        "    #a = 'indian'\n",
        "    for w in mf_accents[a].keys():\n",
        "        acc[0].append(sum(max_accents[a][w][0])/len(max_accents[a][w][0]))\n",
        "        acc[1].append(sum(max_accents[a][w][1])/len(max_accents[a][w][1]))\n",
        "        #acc[2].append(np.asarray(max_accents[a][w][2]).mean())\n",
        "    print(np.asarray(acc[0]).mean(),np.asarray(acc[1]).mean(),np.asarray(acc[2]).mean())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DOPUpZkBRF-B"
      },
      "outputs": [],
      "source": [
        "# us\n",
        "# 0.822280626362392 0.7107428299869577\n",
        "# canada\n",
        "# 0.8579726753622985 0.7690201257461524\n",
        "# indian\n",
        "# 0.7958703395891465 0.6934186392183957\n",
        "# african\n",
        "# 0.8308889379878893 0.7204420312446458\n",
        "# england\n",
        "# 0.7781214935118137 0.7200572594990948\n",
        "# scotland\n",
        "# 0.8124665953005906 0.7094160821071285\n",
        "# australia\n",
        "# 0.7892047076340786 0.7292303868344401"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mi89lTs5RF-D"
      },
      "outputs": [],
      "source": [
        "def cross_entropy(predictions, targets,N):\n",
        "    \"\"\"\n",
        "    Computes cross entropy between targets (encoded as one-hot vectors)\n",
        "    and predictions. \n",
        "    Input: predictions (N, k) ndarray\n",
        "           targets (N, k) ndarray        \n",
        "    Returns: scalar\n",
        "    \"\"\"\n",
        "    #predictions = np.clip(predictions, epsilon, 1. - epsilon)\n",
        "    #N = predictions.shape[0]\n",
        "    ce = -np.sum(targets*np.log(predictions+1e-9))/N\n",
        "    return ce\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ZOm-uXrRF-D"
      },
      "source": [
        "### EMD Calculations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lqHSoz4URF-E"
      },
      "source": [
        "##### EMD & entropy wrt baselines at word level. Wd1 and e1 correspond to emd and entropy between the segment of at attribution of frame corresponding to the word and a uniformly distributed baseline for the duration of the word respectively. Wd2 and e2 correspond to emd and entropy between the attribution of the entire transcription and a uniformly distributed baseline for the duration of the word and zero every where else respectively"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "THzZA9X-RF-E"
      },
      "outputs": [],
      "source": [
        "def grad_word_dist(file, word_idx, actual_idx, taper = False):\n",
        "    \n",
        "    #try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "        #print(file)\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "    with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "        file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "\n",
        "#         print(file_attr['output'].split(' '))\n",
        "    normalized_attr = get_norm_attr(file_attr['grad_dict'])\n",
        "    keys = list(normalized_attr.keys())\n",
        "#         print(keys)\n",
        "    spaces = get_space(file_attr['output'])\n",
        "    indices = [keys.index(x) for x in spaces]\n",
        "    words = get_words(keys,indices, spaces)\n",
        "    input_size = len(file_attr['grad_dict'][list(file_attr['grad_dict'].keys())[0]]) # calculate properly once\n",
        "    buffer = 10\n",
        "    plot_vertical = False\n",
        "    #word_idx = 8\n",
        "#         print(words)\n",
        "#         print(keys)\n",
        "\n",
        "    word_activation = np.sum(np.asarray([np.abs(normalized_attr[idx]) for idx in words[word_idx]] ), axis = 0)\n",
        "    \n",
        "\n",
        "\n",
        "    str_list = []\n",
        "    for x in words[word_idx]:\n",
        "        str_list.append(file_attr['output'][x])\n",
        "\n",
        "\n",
        "    #print(''.join(str_list))\n",
        "    allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "    wrd_allignments,w_start,w_end = get_word_allignment(file, input_size)\n",
        "    wrd_allignments = np.asarray(wrd_allignments)\n",
        "    allignments = np.asarray(allignments)            \n",
        "    actual_size = len(allignments)\n",
        "    \n",
        "    modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "    assert len(wrd_allignments) == len(allignments), \"dimensions don't match\"\n",
        "    modified_allignments = allignments[w_start - buffer:w_end + buffer]\n",
        "    modified_wrd_allignments = wrd_allignments[w_start - buffer:w_end + buffer]\n",
        "    \n",
        "    wrd_labels = [list(x[1]) for x in groupby(modified_wrd_allignments)]\n",
        "#     print(wrd_labels)\n",
        "    wrd_indices = [0]\n",
        "    for j in wrd_labels:\n",
        "        wrd_indices.append(wrd_indices[-1] + len(j))\n",
        "#     print(wrd_indices)\n",
        "    my_arr = word_activation[w_start - buffer:w_end + buffer]/np.sum(word_activation[w_start - buffer:w_end + buffer])\n",
        "    m = actual_idx + 1\n",
        "#     print(m)\n",
        "    \n",
        "    assert len(my_arr) == len(modified_wrd_allignments), 'assumption failed'\n",
        "    word_frame = my_arr[wrd_indices[m]:wrd_indices[m+1]]\n",
        "    word_frame_norm = word_frame/np.sum(word_frame)\n",
        "    baseline_frame_ = np.ones(len(word_frame))/len(word_frame)\n",
        "    baseline_frame = np.array(signal.tukey(int(2*len(word_frame))))\n",
        "    if(not taper):\n",
        "        baseline_frame = baseline_frame_\n",
        "#     print('here')\n",
        "#     print(len(baseline_frame))\n",
        "#     print(len(baseline_wind[wrd_indices[m]:wrd_indices[m+1]]))\n",
        "\n",
        "    baseline_frame = baseline_frame / sum(baseline_frame)\n",
        "#     print(len(baseline_frame))\n",
        "#     print(wrd_indices[m+1] - wrd_indices[m])\n",
        "    count = len(set(modified_allignments[wrd_indices[m]:wrd_indices[m+1]]))#huersitic\n",
        "    (wd1,e1) = 100*wd(word_frame_norm,baseline_frame_)/count, cross_entropy(word_frame_norm,baseline_frame_,count)\n",
        "    baseline_wind = np.zeros(len(my_arr))\n",
        "    if(not taper):\n",
        "        baseline_wind[wrd_indices[m] :wrd_indices[m+1] ] = baseline_frame\n",
        "    else : \n",
        "        if(wrd_indices[m] - len(word_frame)//2 >= 0 and wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 < len(my_arr)):\n",
        "            baseline_wind[wrd_indices[m] - len(word_frame)//2 :wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 ] = baseline_frame\n",
        "        elif(wrd_indices[m] - len(word_frame)//2 < 0):\n",
        "            baseline_wind[0:wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 ] = baseline_frame[-wrd_indices[m] + len(word_frame)//2]\n",
        "        else:\n",
        "            dist = len(my_arr) - (wrd_indices[m+1] + len(word_frame) - len(word_frame)//2)\n",
        "            baseline_wind[wrd_indices[m] - len(word_frame)//2 : len(my_arr) ] = baseline_frame[:len(word_frame) - dist]\n",
        "            \n",
        "            \n",
        "    #my_arr = my_arr/sum(my_arr)\n",
        "#     plt.plot(my_arr)\n",
        "# #     print(round(sum(my_arr),2))\n",
        "#     plt.plot(baseline_wind)\n",
        "# #     print(round(sum(baseline_wind),2))\n",
        "#     plt.show()\n",
        "    #print(wrd_indices)\n",
        "    \n",
        "    #\n",
        "    \n",
        "#     (wd2, e2) = 100*wd(my_arr,baseline_wind)/len(my_arr), cross_entropy(my_arr,baseline_wind,len(my_arr))\n",
        "    bins = np.arange(len(my_arr))\n",
        "    euc_dist = ed(bins.reshape(-1,1), bins.reshape(-1,1))\n",
        "    (wd2, e2) = 100*wd(my_arr,baseline_wind), emd(my_arr.astype(np.float64),baseline_wind.astype(np.float64), euc_dist.astype(np.float64))\n",
        "    return wd1, e1, wd2,e2\n",
        "    \n",
        "    \n",
        "    \n",
        "#     wrd_combined_wonorm = []\n",
        "#     wrd_combined = []\n",
        "#     for m in range(len(wrd_labels)):\n",
        "#         wrd_combined_wonorm.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]]))\n",
        "#         wrd_combined.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]])/len(wrd_labels[m]))\n",
        "#     wrd_combined = np.asarray(wrd_combined)\n",
        "\n",
        "#     wrd_combined = wrd_combined/np.sum(wrd_combined)\n",
        "#     print(''.join(str_list))\n",
        "\n",
        "    #return wrd_combined, wrd_combined_wonorm\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-m_4bQd_RF-E"
      },
      "source": [
        "##### EMD & entropy wrt baselines at syllable level. (wd1, e1), (wd2, e2) represent the same things as above"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VYG4hIXxRF-F"
      },
      "outputs": [],
      "source": [
        "def grad_syll_dist(file, word_idx, actual_idx, syll_num, taper = False):\n",
        "    \n",
        "    #try:\n",
        "        #file = 'common_voice_en_110121'\n",
        "        #print(file)\n",
        "#         Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#         display(Audio(wav, rate=Fs))\n",
        "    with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "        file_attr = pickle.load(j)\n",
        "#         print(file_attr['output'], file_meta[file]['accent'])\n",
        "\n",
        "#     print(file_attr['output'])\n",
        "    normalized_attr = get_norm_attr(file_attr['grad_dict'])\n",
        "    keys = list(normalized_attr.keys())\n",
        "    #print(keys)\n",
        "    sent = file_attr['output'].replace('_','').lower()\n",
        "    chunks = {}\n",
        "    for chunk in sent.split():\n",
        "        chunks[chunk.upper()] = hyphenate_word(chunk)\n",
        "        #print(syllables[chunk.upper()])\n",
        "    #chunks = [x.upper() for x in chunks]\n",
        "    #print(chunks)\n",
        "    mod_sent = ' '.join(chunks)\n",
        "    spaces = get_space(file_attr['output']) \n",
        "    #print(spaces)\n",
        "    indices = [keys.index(x) for x in spaces]\n",
        "    #print(indices)\n",
        "    words = get_words(keys,indices, spaces)\n",
        "    #print(words)\n",
        "    w_new = []\n",
        "    mult_syll = False\n",
        "    for w in words:\n",
        "        if(len(w) ==1 and w[0] in spaces):\n",
        "            w_new.append(w)\n",
        "            continue\n",
        "        #print(w)\n",
        "        can = ''.join([file_attr['output'][i] for i in w])\n",
        "        if(len(chunks[can]) == 1): w_new.append(w)\n",
        "        else:\n",
        "#             mult_syll = True\n",
        "            Inputt = iter(w)\n",
        "            length_to_split = [len(i) for i in chunks[can]]\n",
        "            Output = [list(islice(Inputt, elem)) for elem in length_to_split]\n",
        "            w_new.extend(Output)\n",
        "    #print(w_new)\n",
        "#     print(mult_syll)   \n",
        "    input_size = len(file_attr['grad_dict'][list(file_attr['grad_dict'].keys())[0]]) # calculate properly once\n",
        "    buffer = 10\n",
        "    plot_vertical = False\n",
        "    #word_idx = 8\n",
        "#         print(words)\n",
        "#         print(keys)\n",
        "    syll_idx = word_idx + syll_num\n",
        "    #print(w_new[syll_idx])\n",
        "    word_activation = np.sum(np.asarray([np.abs(normalized_attr[idx]) for idx in w_new[syll_idx]] ), axis = 0)\n",
        "    \n",
        "\n",
        "\n",
        "    str_list = []\n",
        "\n",
        "    for x in w_new[syll_idx]:\n",
        "        str_list.append(file_attr['output'][x])\n",
        "\n",
        "\n",
        "#     print(''.join(str_list))\n",
        "    allignments, p_start, p_end = get_frame_allignment(file, input_size)\n",
        "    wrd_allignments,w_start,w_end = get_word_allignment(file, input_size)\n",
        "    #print('here',allignments)\n",
        "    wrd_allignments = np.asarray(wrd_allignments)\n",
        "    allignments = np.asarray(allignments)            \n",
        "    actual_size = len(allignments)\n",
        "    \n",
        "    modified_allignments = allignments[p_start - buffer:p_end + buffer]\n",
        "    assert len(wrd_allignments) == len(allignments), \"dimensions don't match\"\n",
        "    modified_allignments = allignments[w_start - buffer:w_end + buffer]\n",
        "    modified_wrd_allignments = wrd_allignments[w_start - buffer:w_end + buffer]\n",
        "#     print(modified_allignments)\n",
        "#     print(modified_wrd_allignments)\n",
        "    \n",
        "    \n",
        "    wrd_labels = [list(x[1]) for x in groupby(modified_wrd_allignments)]\n",
        "#     print(wrd_labels)\n",
        "    wrd_indices = [0]\n",
        "    for j in wrd_labels:\n",
        "        wrd_indices.append(wrd_indices[-1] + len(j))\n",
        "#     print(wrd_indices)\n",
        "\n",
        "    my_arr = word_activation[w_start - buffer:w_end + buffer]/np.sum(word_activation[w_start - buffer:w_end + buffer])\n",
        "    m = actual_idx + 1\n",
        "#     print(m)\n",
        "#     print(modified_allignments[wrd_indices[m] :wrd_indices[m+1]])\n",
        "    \n",
        "    #target_word = wrd_labels[a]\n",
        "    #print()\n",
        "    #print(chunks[])\n",
        "    items = modified_allignments[wrd_indices[m] :wrd_indices[m+1]]\n",
        "    my_pron = list(OrderedDict.fromkeys(items))\n",
        "    my_pron = ' '.join(my_pron).upper()\n",
        "#     print(my_pron)\n",
        "    my_syll = syllables[wrd_labels[m][0]]\n",
        "    if(len(my_syll) > 1): mult_syll = True\n",
        "    flag = True\n",
        "    item_labels = [list(x[1]) for x in groupby(items)]\n",
        "    item_lens = [len(x) for x in item_labels]\n",
        "    for i in range(len(syllables[wrd_labels[m][0]])):\n",
        "        if (my_syll[i]['pron'] == my_pron):\n",
        "            flag = False\n",
        "            assert len(my_syll[i]['syll']) == len(hyphenate_word(wrd_labels[m][0])), 'syll-hyph failed'\n",
        "            phn_splits = []\n",
        "            for j in my_syll[i]['syll']:\n",
        "                j = [k for k in j if len(k)!=0 ]\n",
        "#                 print(j)\n",
        "                phn_splits.append(len(j))\n",
        "    Inputt = iter(item_lens)\n",
        "    length_to_split = phn_splits\n",
        "    Output = [list(islice(Inputt, elem)) for elem in length_to_split]\n",
        "    syll_indices = [0]\n",
        "    syll_indices.extend([sum(j) for j in Output])\n",
        "#     print(syll_indices)\n",
        "#     print(wrd_indices[m], wrd_indices[m+1])\n",
        "    assert not flag, 'no syllables'\n",
        "    \n",
        "#     print(Output)\n",
        "#     print(item_lens)    \n",
        "    assert len(my_arr) == len(modified_wrd_allignments), 'assumption failed'\n",
        "    word_frame = my_arr[wrd_indices[m]:wrd_indices[m+1]]\n",
        "    word_frame_norm = word_frame/np.sum(word_frame)\n",
        "    baseline_frame_ = np.ones(len(word_frame))/len(word_frame)\n",
        "    baseline_frame = np.array(signal.tukey(int(2*len(word_frame))))\n",
        "    if(not taper):\n",
        "        baseline_frame = baseline_frame_\n",
        "#     print('here')\n",
        "#     print(len(baseline_frame))\n",
        "#     print(len(baseline_wind[wrd_indices[m]:wrd_indices[m+1]]))\n",
        "\n",
        "    baseline_frame = baseline_frame / sum(baseline_frame)\n",
        "#     print(len(baseline_frame))\n",
        "#     print(wrd_indices[m+1] - wrd_indices[m])\n",
        "    count = len(set(modified_allignments[wrd_indices[m]:wrd_indices[m+1]]))#huersitic\n",
        "    (wd1,e1) = 100*wd(word_frame_norm,baseline_frame_)/count, cross_entropy(word_frame_norm,baseline_frame_,count)\n",
        "    baseline_wind = np.zeros(len(my_arr))\n",
        "    if(not taper):\n",
        "        if( not mult_syll):\n",
        "            baseline_wind[wrd_indices[m] :wrd_indices[m+1] ] = baseline_frame\n",
        "        else:\n",
        "            #print()\n",
        "            baseline_syll = np.ones(syll_indices[syll_num +1 ] - syll_indices[syll_num])\n",
        "            baseline_syll = baseline_syll/len(baseline_syll)\n",
        "            baseline_wind[wrd_indices[m] + syll_indices[syll_num] : wrd_indices[m] + syll_indices[syll_num + 1] ] = baseline_syll\n",
        "            \n",
        "            \n",
        "    else : \n",
        "        if(wrd_indices[m] - len(word_frame)//2 >= 0 and wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 < len(my_arr)):\n",
        "            baseline_wind[wrd_indices[m] - len(word_frame)//2 :wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 ] = baseline_frame\n",
        "        elif(wrd_indices[m] - len(word_frame)//2 < 0):\n",
        "            baseline_wind[0:wrd_indices[m+1] + len(word_frame) - len(word_frame)//2 ] = baseline_frame[-wrd_indices[m] + len(word_frame)//2]\n",
        "        else:\n",
        "            dist = len(my_arr) - (wrd_indices[m+1] + len(word_frame) - len(word_frame)//2)\n",
        "            baseline_wind[wrd_indices[m] - len(word_frame)//2 : len(my_arr) ] = baseline_frame[:len(word_frame) - dist]\n",
        "            \n",
        "            \n",
        "    #my_arr = my_arr/sum(my_arr)\n",
        "#     plt.plot(my_arr)\n",
        "# #     print(round(sum(my_arr),2))\n",
        "#     plt.plot(baseline_wind)\n",
        "# #     print(round(sum(baseline_wind),2))\n",
        "#     plt.show()\n",
        "    #print(wrd_indices)\n",
        "    \n",
        "    #\n",
        "    \n",
        "#     (wd2, e2) = 100*wd(my_arr,baseline_wind)/len(my_arr), cross_entropy(my_arr,baseline_wind,len(my_arr))\n",
        "    bins = np.arange(len(my_arr))\n",
        "    euc_dist = ed(bins.reshape(-1,1), bins.reshape(-1,1))\n",
        "    (wd2, e2) = 100*wd(my_arr,baseline_wind), emd(my_arr.astype(np.float64),baseline_wind.astype(np.float64), euc_dist.astype(np.float64))\n",
        "    return wd1, e1, wd2,e2\n",
        "    \n",
        "    \n",
        "    \n",
        "#     wrd_combined_wonorm = []\n",
        "#     wrd_combined = []\n",
        "#     for m in range(len(wrd_labels)):\n",
        "#         wrd_combined_wonorm.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]]))\n",
        "#         wrd_combined.append(np.sum(my_arr[wrd_indices[m]:wrd_indices[m+1]])/len(wrd_labels[m]))\n",
        "#     wrd_combined = np.asarray(wrd_combined)\n",
        "\n",
        "#     wrd_combined = wrd_combined/np.sum(wrd_combined)\n",
        "#     print(''.join(str_list))\n",
        "\n",
        "    #return wrd_combined, wrd_combined_wonorm\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nx8uHX7QRF-F"
      },
      "outputs": [],
      "source": [
        "grad_syll_dist('common_voice_en_179645',2,1,0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p8QqNPEeRF-F"
      },
      "outputs": [],
      "source": [
        "\n",
        "mf_us = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_canada = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_indian = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_african = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_england = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_scotland = {x:([],[],[],[]) for x in most_frequent}\n",
        "mf_australian = {x:([],[],[],[]) for x in most_frequent}\n",
        "dist_accents = {'us':mf_us,'canada':mf_canada,'indian':mf_indian,'african':mf_african,'england':mf_england,'scotland':mf_scotland,'australia':mf_australian}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4kWnkRCuRF-G"
      },
      "outputs": [],
      "source": [
        "# print(target_transcript)\n",
        "# for file in trans_dict[target_transcript]:\n",
        "#     Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#     display(Audio(wav, rate=Fs))\n",
        "#     with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "#         file_attr = pickle.load(j)\n",
        "#     print(file_attr['output'], file_meta[file]['accent'])\n",
        "#     print(list(file_attr['attr dict'].keys()))\n",
        "#     #inp_grad_grapheme(file, grapheme_idx)\n",
        "#     print(file)\n",
        "#     print(grad_word_dist(file,2, 1))\n",
        "#     break\n",
        "valid_punctuation = string.punctuation.replace(\"'\",\"\")\n",
        "for t in transcripts:\n",
        "#     print(t)\n",
        "    \n",
        "    t_ = t.strip().upper()\n",
        "    t_ = t_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    #print(t_)\n",
        "    t_list = t_.split()\n",
        "    true_nz_counts = {x:t_list.count(x) for x in most_frequent if t_list.count(x) > 0}\n",
        "    true_nz_indices = {x:[index for index, value in enumerate(t_list) if value == x] for x in true_nz_counts.keys()}\n",
        "    \n",
        "    for file in trans_dict[t]:\n",
        "        \n",
        "        try:   \n",
        "            with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "                file_attr = pickle.load(j)\n",
        "            op = file_attr['output'].replace('_', '')\n",
        "    #             print(op)\n",
        "            op_list = op.split()\n",
        "            accent = file_meta[file]['accent']\n",
        "            #print(accent,'------------------')\n",
        "            a_indices = {x:[index for index, value in enumerate(op_list) if value == x] for x in true_nz_counts.keys()}\n",
        "\n",
        "            for w in true_nz_counts.keys():\n",
        "                for idx in a_indices[w]:\n",
        "    #                     print('here')\n",
        "    #                     print(w)\n",
        "                    # compute condition for correctness\n",
        "                    distr, distr_wo = grad_clubbed_word(file, 2*idx, disable_prints=True)\n",
        "                    #print(distr)\n",
        "                    max_idx = np.argmax(np.asarray(distr))\n",
        "                    max_idx_wo = np.argmax(np.asarray(distr_wo))\n",
        "                    if(is_correct(op,t_,idx)):\n",
        "    #                         print('lol')\n",
        "    #                         print(true_nz_indices[w])\n",
        "    #                         print(a_indices[w].index(idx))\n",
        "                        idx2 = true_nz_indices[w][a_indices[w].index(idx)]\n",
        "                        wd1,e1,wd2,e2 = grad_word_dist(file,2*idx,idx2, taper= False)\n",
        "                        dist_accents[accent][w][0].append(wd1)\n",
        "                        dist_accents[accent][w][1].append(e1)\n",
        "                        if(np.isnan(e2/len(w))):\n",
        "                           print('encountered nan', e2, len(w))\n",
        "                           continue\n",
        "                        dist_accents[accent][w][2].append(e2/len(w))\n",
        "                        #print(wd2, e2)\n",
        "                        dist_accents[accent][w][3].append(e2)\n",
        "        except:\n",
        "            continue\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CpuwkyaXRF-G"
      },
      "outputs": [],
      "source": [
        "\n",
        "mf_us = {x:([],[],[],[]) for x in chosen}\n",
        "mf_canada = {x:([],[],[],[]) for x in chosen}\n",
        "mf_indian = {x:([],[],[],[]) for x in chosen}\n",
        "mf_african = {x:([],[],[],[]) for x in chosen}\n",
        "mf_england = {x:([],[],[],[]) for x in chosen}\n",
        "mf_scotland = {x:([],[],[],[]) for x in chosen}\n",
        "mf_australian = {x:([],[],[],[]) for x in chosen}\n",
        "dist_accents = {'us':mf_us,'canada':mf_canada,'indian':mf_indian,'african':mf_african,'england':mf_england,'scotland':mf_scotland,'australia':mf_australian}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ToJ9Vx5ZRF-G"
      },
      "outputs": [],
      "source": [
        "# print(target_transcript)\n",
        "# for file in trans_dict[target_transcript]:\n",
        "#     Fs, wav = wavfile.read('my_data/MCV_validated_{}/wav/{}.wav'.format(file_meta[file]['accent'],file))\n",
        "#     display(Audio(wav, rate=Fs))\n",
        "#     with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "#         file_attr = pickle.load(j)\n",
        "#     print(file_attr['output'], file_meta[file]['accent'])\n",
        "#     print(list(file_attr['attr dict'].keys()))\n",
        "#     #inp_grad_grapheme(file, grapheme_idx)\n",
        "#     print(file)\n",
        "#     print(grad_word_dist(file,2, 1))\n",
        "#     break\n",
        "valid_punctuation = string.punctuation.replace(\"'\",\"\")\n",
        "most_frequent = chosen_words\n",
        "for t in transcripts:\n",
        "    \n",
        "    t_ = t.strip().upper()\n",
        "    t_ = t_.translate(str.maketrans({a:None for a in valid_punctuation }))\n",
        "    #print(t_)\n",
        "    t_list = t_.split()\n",
        "    true_nz_counts = {x:t_list.count(x) for x in most_frequent if t_list.count(x) > 0}\n",
        "    true_nz_indices = {x:[index for index, value in enumerate(t_list) if value == x] for x in true_nz_counts.keys()}\n",
        "    \n",
        "    for file in trans_dict[t]:\n",
        "        \n",
        "        try:   \n",
        "            with open('attribution/grad/{}.pickle'.format(file), 'rb') as j:\n",
        "                file_attr = pickle.load(j)\n",
        "            op = file_attr['output'].replace('_', '')\n",
        "    #             print(op)\n",
        "            op_list = op.split()\n",
        "            accent = file_meta[file]['accent']\n",
        "            #print(accent,'------------------')\n",
        "            a_indices = {x:[index for index, value in enumerate(op_list) if value == x] for x in true_nz_counts.keys()}\n",
        "\n",
        "            for w in true_nz_counts.keys():\n",
        "                for idx in a_indices[w]:\n",
        "    #                     print('here')\n",
        "    #                     print(w)\n",
        "                    # compute condition for correctness\n",
        "                    distr, distr_wo = grad_clubbed_word(file, 2*idx, disable_prints=True)\n",
        "                    #print(distr)\n",
        "                    max_idx = np.argmax(np.asarray(distr))\n",
        "                    max_idx_wo = np.argmax(np.asarray(distr_wo))\n",
        "                    if(is_correct(op,t_,idx)):\n",
        "    #                         print('lol')\n",
        "    #                         print(true_nz_indices[w])\n",
        "    #                         print(a_indices[w].index(idx))\n",
        "                        idx2 = true_nz_indices[w][a_indices[w].index(idx)]\n",
        "                        for l in range(len(hyphenate_word(w))):\n",
        "                            wd1,e1,wd2,e2 = grad_syll_dist(file,2*idx,idx2,l, taper= False)\n",
        "#                             dist_accents[accent][w][0].append(wd1)\n",
        "#                         dist_accents[accent][w][1].append(e1)\n",
        "                            if(np.isnan(e2)):\n",
        "                               print('encountered nan', e2)\n",
        "                               continue\n",
        "                            #dist_accents[accent][w][2].append(e2/len(w))\n",
        "                            #print(wd2, e2)\n",
        "                            #print(dist_accents[accent])\n",
        "                            #print([hyphenate_word(w)[l]])\n",
        "                            dist_accents[accent][hyphenate_word(w)[l]][3].append(e2)\n",
        "                            #print(dist_accents[accent])\n",
        "                            #break\n",
        "                    \n",
        "        except:\n",
        "            continue\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FBdUmeFJRF-H",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "acc_chosen = {k:{a:0 for a in dist_accents.keys()} for k in chosen}\n",
        "for a in dist_accents.keys():\n",
        "#     print(a)\n",
        "    #print(chosen)\n",
        "    acc = {k:[] for k in chosen}\n",
        "    #a = 'indian'\n",
        "    #print(mf_accents[a].keys())\n",
        "    for w in chosen:\n",
        "        #print(len(acc[2]))\n",
        "        #print(len(dist_accents[a][w][2]))\n",
        "#         acc[0].append(np.asarray(dist_accents[a][w][0]).mean())\n",
        "#         acc[1].append(np.asarray(dist_accents[a][w][1]).mean())\n",
        "        inter = np.asarray(dist_accents[a][w][3]).mean()\n",
        "        if(not np.isnan(inter)):\n",
        "            acc[w].append(inter)\n",
        "            acc_chosen[w][a] = inter\n",
        "        #acc[3].append(np.asarray(dist_accents[a][w][3]).mean())\n",
        "    \n",
        "#         acc[word_cluster[w]].extend(np.random.choice(np.asarray(dist_accents[a][w][2]),int(min_dist[most_frequent.index(w)]), replace = False))\n",
        "#         acc[word_cluster[w]].append(np.random.choice(np.asarray(dist_accents[a][w][2]),int(min_dist[most_frequent.index(w)]), replace = False).mean())\n",
        "#         acc[3].append(np.asarray(dist_accents[a][w][3]).mean())\n",
        "    #print(len(np.asarray(acc[2])))\n",
        "#     print(a, acc)\n",
        "#     print('------------------------')\n",
        "print('___________________________')\n",
        "for l in acc_chosen.keys():\n",
        "    print(l, acc_chosen[l])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U9QszI8FRF-H"
      },
      "source": [
        "### <font color=\"00ff00\">  **Clustering words based on number of phones**</font>\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CZDKUcRbRF-H"
      },
      "outputs": [],
      "source": [
        "nltk.download('cmudict')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jYNQZEDxRF-I"
      },
      "outputs": [],
      "source": [
        "def get_cluster(num):\n",
        "    if(num < 3):\n",
        "        return 0\n",
        "    elif(num < 4):\n",
        "        return 1\n",
        "    else: return 2\n",
        "# def get_cluster(num):\n",
        "#     if(num < 3):\n",
        "#         return 0\n",
        "#     elif(num < 5):\n",
        "#         return 1\n",
        "#     else: return 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iJinO0HaRF-I",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "arpabet = nltk.corpus.cmudict.dict()\n",
        "word_phn = np.zeros(len(most_frequent))\n",
        "word_len = np.zeros(len(most_frequent))\n",
        "for w in most_frequent:\n",
        "    word_phn[most_frequent.index(w)] = len(arpabet[w.lower()][0])\n",
        "    #break\n",
        "print(word_phn)\n",
        "for w in most_frequent:\n",
        "    word_len[most_frequent.index(w)] = len(w)\n",
        "    #break\n",
        "# print(word_phn)\n",
        "print(word_len)\n",
        "print(most_frequent)\n",
        "word_cluster = {x: get_cluster(word_phn[most_frequent.index(x)]) for x in most_frequent}\n",
        "print(word_cluster)\n",
        "print(word_phn[most_frequent.index(w)])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "72pV6nDORF-I"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f4frXtXXRF-I"
      },
      "outputs": [],
      "source": [
        "min_dist = np.ones(len(most_frequent))*10000\n",
        "for w in most_frequent:\n",
        "    #print(w)\n",
        "    \n",
        "    for a in dist_accents.keys():\n",
        "       \n",
        "        min_dist[most_frequent.index(w)] = min(len(dist_accents[a][w][2]), min_dist[most_frequent.index(w)])\n",
        "        \n",
        "print(min_dist)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gqtJva_5RF-J"
      },
      "source": [
        "##### <font color=white> **Report EMD based on wd2**</font> \n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OuD5Sls9RF-J"
      },
      "outputs": [],
      "source": [
        "for a in dist_accents.keys():\n",
        "#     print(a)\n",
        "    acc = ([],[],[],[])\n",
        "    #a = 'indian'\n",
        "    for w in mf_accents[a].keys():\n",
        "        #print(len(acc[2]))\n",
        "        #print(len(dist_accents[a][w][2]))\n",
        "#         acc[0].append(np.asarray(dist_accents[a][w][0]).mean())\n",
        "#         acc[1].append(np.asarray(dist_accents[a][w][1]).mean())\n",
        "        acc[word_cluster[w]].append(np.asarray(dist_accents[a][w][3]).mean())\n",
        "        acc[3].append(np.asarray(dist_accents[a][w][3]).mean())\n",
        "    \n",
        "#         acc[word_cluster[w]].extend(np.random.choice(np.asarray(dist_accents[a][w][2]),int(min_dist[most_frequent.index(w)]), replace = False))\n",
        "#         acc[word_cluster[w]].append(np.random.choice(np.asarray(dist_accents[a][w][2]),int(min_dist[most_frequent.index(w)]), replace = False).mean())\n",
        "#         acc[3].append(np.asarray(dist_accents[a][w][3]).mean())\n",
        "    #print(len(np.asarray(acc[2])))\n",
        "    print(a, np.asarray(acc[0]).mean().round(2), np.asarray(acc[1]).mean().round(2), np.asarray(acc[2]).mean().round(2), np.asarray(acc[3]).mean().round(2))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hBAsy88qRF-J"
      },
      "source": [
        "##### <font color=white> **Report EMD based on wd2**</font> \n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nXsD894xRF-J"
      },
      "outputs": [],
      "source": [
        "with open('syll.pickle','rb') as s:\n",
        "    syllables = pickle.load(s)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lCdpUf_2RF-J"
      },
      "outputs": [],
      "source": [
        "syll_dict = {}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Av8Nk5DvRF-K"
      },
      "outputs": [],
      "source": [
        "count = 0\n",
        "seg = 0\n",
        "for w in syllables.keys():\n",
        "    \n",
        "    \n",
        "    wl = w.lower()\n",
        "    hyp = hyphenate_word(wl)\n",
        "    for h in hyp:\n",
        "        if(h not in syll_dict.keys()):\n",
        "            syll_dict[h]={'count':my_freq[my_freq_words.index(w)][1],'words':[w]}\n",
        "        else:\n",
        "            syll_dict[h]['count'] += my_freq[my_freq_words.index(w)][1]\n",
        "            syll_dict[h]['words'].append(w)\n",
        "            \n",
        "    try:\n",
        "        if(len(syllables[w][0]['syll'] ) == len(hyp)):\n",
        "            seg += len(hyp)\n",
        "            syllables[w][0]['splt'] = hyp\n",
        "        else:\n",
        "            count+=1\n",
        "            print(w, count)\n",
        "    except:\n",
        "        count += 1\n",
        "        print(w)\n",
        "#print(seg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "17vFFysGRF-K"
      },
      "outputs": [],
      "source": [
        "print(syll_dict)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DzXrrshrRF-M"
      },
      "outputs": [],
      "source": [
        "chosen = [k.upper() for k, v in sorted(syll_dict.items(), key=lambda item: item[1]['count'])][::-1][:75]\n",
        "print(chosen)\n",
        "chosen_words = []\n",
        "for c in chosen:\n",
        "    chosen_words.extend(syll_dict[c.lower()]['words'] )\n",
        "chosen_words = list(set(chosen_words))\n",
        "print(chosen_words)\n",
        "print(len(chosen_words))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EE6l-kglRF-M"
      },
      "outputs": [],
      "source": [
        "syllables['CREATIVE']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bC7LGSA3RF-M"
      },
      "outputs": [],
      "source": [
        "syll_dict['minder']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QLrTAWw5RF-N"
      },
      "outputs": [],
      "source": [
        "len(list(syllables.keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "skar_VOHRF-N"
      },
      "outputs": [],
      "source": [
        "my_freq = allWordDist.most_common(1256)\n",
        "my_freq_words = [w[0] for w in my_freq]"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "I3BH52cNDL94"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}