UrduRNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "UrduRNN.ipynb",
      "provenance": [],
      "mount_file_id": "1f-Bal1BNWYGN0oMPjJpt5g4WdABO0n8_",
      "authorship_tag": "ABX9TyPPtltu4zFotcyuP9bFWro4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "j5WvFlEsLTOZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os \n",
        "os.chdir('drive/My Drive/Urdu2')"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oa44TZyozjB0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.chdir('nmt-keras')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q712shVEGBy4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c19f0163-4898-4e86-932b-5bd8a0d54e46"
      },
      "source": [
        "!pip install -e .\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Obtaining file:///content/drive/My%20Drive/Urdu2/nmt-keras\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.3.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (0.16.0)\n",
            "Requirement already satisfied: keras@ https://github.com/MarcBS/keras/archive/master.zip from https://github.com/MarcBS/keras/archive/master.zip in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (2.3.1.1)\n",
            "Requirement already satisfied: keras_applications in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.0.8)\n",
            "Requirement already satisfied: keras_preprocessing in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.1.2)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (2.10.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (3.2.2)\n",
            "Requirement already satisfied: multimodal-keras-wrapper in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (3.1.6)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.18.5)\n",
            "Requirement already satisfied: scikit-image in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (0.16.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (0.22.2.post1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.12.0)\n",
            "Requirement already satisfied: tables in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (3.4.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.0.5)\n",
            "Requirement already satisfied: sacrebleu in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.4.10)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (0.0.43)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.4.1)\n",
            "Requirement already satisfied: tensorflow<2 in /usr/local/lib/python3.6/dist-packages (from nmt-keras==0.6) (1.15.3)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras@ https://github.com/MarcBS/keras/archive/master.zip->nmt-keras==0.6) (3.13)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->nmt-keras==0.6) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->nmt-keras==0.6) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->nmt-keras==0.6) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->nmt-keras==0.6) (2.4.7)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (from multimodal-keras-wrapper->nmt-keras==0.6) (0.29.20)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.6/dist-packages (from multimodal-keras-wrapper->nmt-keras==0.6) (0.10.0)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from multimodal-keras-wrapper->nmt-keras==0.6) (0.0)\n",
            "Requirement already satisfied: subword-nmt in /usr/local/lib/python3.6/dist-packages (from multimodal-keras-wrapper->nmt-keras==0.6) (0.3.7)\n",
            "Requirement already satisfied: imageio>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->nmt-keras==0.6) (2.4.1)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->nmt-keras==0.6) (2.4)\n",
            "Requirement already satisfied: pillow>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->nmt-keras==0.6) (7.0.0)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->nmt-keras==0.6) (1.1.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->nmt-keras==0.6) (0.15.1)\n",
            "Requirement already satisfied: numexpr>=2.5.2 in /usr/local/lib/python3.6/dist-packages (from tables->nmt-keras==0.6) (2.7.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->nmt-keras==0.6) (2018.9)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.6/dist-packages (from sacrebleu->nmt-keras==0.6) (1.7.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from sacremoses->nmt-keras==0.6) (2019.12.20)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from sacremoses->nmt-keras==0.6) (4.41.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->nmt-keras==0.6) (7.1.2)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (3.10.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (1.29.0)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (0.2.2)\n",
            "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (1.15.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (0.2.0)\n",
            "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (1.15.1)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (0.34.2)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (1.12.1)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (0.8.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (0.9.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2->nmt-keras==0.6) (3.2.1)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.0->scikit-image->nmt-keras==0.6) (4.4.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow<2->nmt-keras==0.6) (47.3.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2->nmt-keras==0.6) (3.2.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2->nmt-keras==0.6) (1.0.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2->nmt-keras==0.6) (1.6.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2->nmt-keras==0.6) (3.1.0)\n",
            "Installing collected packages: nmt-keras\n",
            "  Found existing installation: nmt-keras 0.6\n",
            "    Can't uninstall 'nmt-keras'. No files were found to uninstall.\n",
            "  Running setup.py develop for nmt-keras\n",
            "Successfully installed nmt-keras\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ygObiEr1LcXJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras_wrapper.dataset import Dataset, saveDataset\n",
        "from data_engine.prepare_data import keep_n_captions\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j38kiWSN2dco",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds = Dataset('task', 'task', silence=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x4An--BGLjEE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "24adfc45-5c77-4f7c-c1af-c92504286afd"
      },
      "source": [
        "ds.setOutput('examples/task/train.en',\n",
        "             'train',\n",
        "             type='text',\n",
        "             id='target_text',\n",
        "             tokenization='tokenize_none',\n",
        "             build_vocabulary=True,\n",
        "             pad_on_batch=True,\n",
        "             sample_weights=True,\n",
        "             max_text_len=50,\n",
        "             max_words=30000,\n",
        "             min_occ=0)\n",
        "\n",
        "ds.setOutput('examples/task/dev.en',\n",
        "             'val',\n",
        "             type='text',\n",
        "             id='target_text',\n",
        "             pad_on_batch=True,\n",
        "             tokenization='tokenize_none',\n",
        "             sample_weights=True,\n",
        "             max_text_len=50,\n",
        "             max_words=0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:22:16] \tApplying tokenization function: \"tokenize_none\".\n",
            "[23/06/2020 17:22:16] Creating vocabulary for data with data_id 'target_text'.\n",
            "[23/06/2020 17:22:16] \t Total: 6284 unique words in 7400 sentences with a total of 192565 words.\n",
            "[23/06/2020 17:22:16] Creating dictionary of 30000 most common words, covering 100.0% of the text.\n",
            "[23/06/2020 17:22:16] Loaded \"train\" set outputs of data_type \"text\" with data_id \"target_text\" and length 7400.\n",
            "[23/06/2020 17:22:16] \tApplying tokenization function: \"tokenize_none\".\n",
            "[23/06/2020 17:22:16] Loaded \"val\" set outputs of data_type \"text\" with data_id \"target_text\" and length 300.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k21_4iMWMTb8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "7c62919e-7c84-4a91-fdde-a4a853b61788"
      },
      "source": [
        "ds.setInput('examples/task/train.ur',\n",
        "            'train',\n",
        "            type='text',\n",
        "            id='source_text',\n",
        "            pad_on_batch=True,\n",
        "            tokenization='tokenize_none',\n",
        "            build_vocabulary=True,\n",
        "            fill='end',\n",
        "            max_text_len=50,\n",
        "            max_words=30000,\n",
        "            min_occ=0)\n",
        "ds.setInput('examples/task/dev.ur',\n",
        "            'val',\n",
        "            type='text',\n",
        "            id='source_text',\n",
        "            pad_on_batch=True,\n",
        "            tokenization='tokenize_none',\n",
        "            fill='end',\n",
        "            max_text_len=50,\n",
        "            min_occ=0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:22:19] \tApplying tokenization function: \"tokenize_none\".\n",
            "[23/06/2020 17:22:19] Creating vocabulary for data with data_id 'source_text'.\n",
            "[23/06/2020 17:22:19] \t Total: 6877 unique words in 7400 sentences with a total of 186176 words.\n",
            "[23/06/2020 17:22:19] Creating dictionary of 30000 most common words, covering 100.0% of the text.\n",
            "[23/06/2020 17:22:19] Loaded \"train\" set inputs of data_type \"text\" with data_id \"source_text\" and length 7400.\n",
            "[23/06/2020 17:22:19] \tApplying tokenization function: \"tokenize_none\".\n",
            "[23/06/2020 17:22:19] Loaded \"val\" set inputs of data_type \"text\" with data_id \"source_text\" and length 300.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dfh1vvLmMam2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "518ec155-5a93-4b05-e155-6bf538dfe60b"
      },
      "source": [
        "ds.setInput('examples/task/train.en',\n",
        "            'train',\n",
        "            type='text',\n",
        "            id='state_below',\n",
        "            required=False,\n",
        "            tokenization='tokenize_none',\n",
        "            pad_on_batch=True,\n",
        "            build_vocabulary='target_text',\n",
        "            offset=1,\n",
        "            fill='end',\n",
        "            max_text_len=50,\n",
        "            max_words=30000)\n",
        "ds.setInput(None,\n",
        "            'val',\n",
        "            type='ghost',\n",
        "            id='state_below',\n",
        "            required=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:22:41] \tApplying tokenization function: \"tokenize_none\".\n",
            "[23/06/2020 17:22:41] \tReusing vocabulary named \"target_text\" for data with data_id \"state_below\".\n",
            "[23/06/2020 17:22:41] Loaded \"train\" set inputs of data_type \"text\" with data_id \"state_below\" and length 7400.\n",
            "[23/06/2020 17:22:41] Loaded \"val\" set inputs of data_type \"ghost\" with data_id \"state_below\" and length 300.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_8axlX_NA27",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "e1a439e0-19d4-4ade-f2c8-a58e056844cd"
      },
      "source": [
        "  for split, input_text_filename in zip(['train', 'val'], ['examples/task/train.ur', 'examples/task/dev.ur']):\n",
        "    ds.setRawInput(input_text_filename,\n",
        "                  split,\n",
        "                  type='file-name',\n",
        "                  id='raw_source_text',\n",
        "                  overwrite_split=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:23:12] Loaded \"train\" set inputs of type \"file-name\" with id \"raw_source_text\".\n",
            "[23/06/2020 17:23:12] Loaded \"val\" set inputs of type \"file-name\" with id \"raw_source_text\".\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cvy6zbesNIV7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "758e606b-a178-4675-f3b2-14597a2e3c78"
      },
      "source": [
        "keep_n_captions(ds, repeat=1, n=1, set_names=['val'])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:23:21] Keeping 1 captions per input on the val set.\n",
            "[23/06/2020 17:23:21] Samples reduced to 300 in val set.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8z6D2WvNKea",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "ed375f43-15f7-45c5-8a6a-6f5dd2f4ec23"
      },
      "source": [
        "saveDataset(ds, 'datasets')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:23:27] <<< creating directory datasets ... >>>\n",
            "[23/06/2020 17:23:27] <<< Saving Dataset instance to datasets/Dataset_task.pkl ... >>>\n",
            "[23/06/2020 17:23:27] <<< Dataset instance saved >>>\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LF7OANQ80n8d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 477
        },
        "outputId": "dda2eb47-391d-4fe1-c857-6716b4454642"
      },
      "source": [
        "!pip install git+https://github.com/MarcBS/keras.git\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/MarcBS/keras.git\n",
            "  Cloning https://github.com/MarcBS/keras.git to /tmp/pip-req-build-56lht_5o\n",
            "  Running command git clone -q https://github.com/MarcBS/keras.git /tmp/pip-req-build-56lht_5o\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (1.18.5)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (1.4.1)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (1.12.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (2.10.0)\n",
            "Requirement already satisfied: keras_applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (1.0.8)\n",
            "Requirement already satisfied: keras_preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras==2.3.1.1) (1.1.2)\n",
            "Building wheels for collected packages: Keras\n",
            "  Building wheel for Keras (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for Keras: filename=Keras-2.3.1.1-cp36-none-any.whl size=487500 sha256=52274eab082cfffe7fd6117e8f4ff02070f18beb56e15da3733d928e59cfc139\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-zz9ymiz6/wheels/83/59/60/d798700bbebbe19766125ac1b5cd87da0dc780bf6b11acecf6\n",
            "Successfully built Keras\n",
            "Installing collected packages: Keras\n",
            "  Found existing installation: Keras 2.3.1\n",
            "    Uninstalling Keras-2.3.1:\n",
            "      Successfully uninstalled Keras-2.3.1\n",
            "Successfully installed Keras-2.3.1.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "keras"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qm1ewrH1NL-1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "47fc86f7-614e-4ad2-e037-97881e3ce2b8"
      },
      "source": [
        "from config import load_parameters\n",
        "from nmt_keras.model_zoo import TranslationModel\n",
        "from keras_wrapper.cnn_model import loadModel\n",
        "from keras_wrapper.dataset import loadDataset\n",
        "from keras_wrapper.extra.callbacks import PrintPerformanceMetricOnEpochEndOrEachNUpdates\n",
        "params = load_parameters()\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n",
            "[24/06/2020 05:36:27] NumExpr defaulting to 2 threads.\n",
            "[24/06/2020 05:36:27] <<< Cupy not available. Using numpy. >>>\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kd4AFP3i0a-f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "803906a4-c1cf-4e05-ab20-a50e5813ac4b"
      },
      "source": [
        "dataset = loadDataset('datasets/Dataset_task.pkl')\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:44:52] <<< Loading Dataset instance from datasets/Dataset_task.pkl ... >>>\n",
            "[24/06/2020 05:44:52] <<< Dataset instance loaded >>>\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUAB2ZAnNRhT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "params['INPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len['source_text']\n",
        "params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len['target_text']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pgZwXUnKNVJe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a7a6531d-6ea4-4aca-9d30-3bac595fc558"
      },
      "source": [
        "params['MODEL_TYPE'] = 'AttentionRNNEncoderDecoder' #  Supported models: 'AttentionRNNEncoderDecoder' and 'Transformer'.\n",
        "nmt_model = TranslationModel(params,\n",
        "                             model_type=params['MODEL_TYPE'], \n",
        "                             model_name='tutorial_model',\n",
        "                             vocabularies=dataset.vocabulary,\n",
        "                             store_path='trained_models/tutorial_model/',\n",
        "                             verbose=True)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:13] <<< Building AttentionRNNEncoderDecoder Translation_Model >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:650: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:13] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:650: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4786: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:13] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4786: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:157: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:13] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:157: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3561: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:14] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3561: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------\n",
            "\t\tTranslationModel instance\n",
            "-----------------------------------------------------------------------------------\n",
            "_model_type: AttentionRNNEncoderDecoder\n",
            "name: tutorial_model\n",
            "model_path: trained_models/tutorial_model/\n",
            "verbose: True\n",
            "\n",
            "Params:\n",
            "\tACCUMULATE_GRADIENTS: 1\n",
            "\tADDITIONAL_OUTPUT_MERGE_MODE: Add\n",
            "\tALIGN_FROM_RAW: True\n",
            "\tALPHA_FACTOR: 0.6\n",
            "\tAMSGRAD: False\n",
            "\tAPPLY_DETOKENIZATION: False\n",
            "\tATTENTION_DROPOUT_P: 0.0\n",
            "\tATTENTION_MODE: add\n",
            "\tATTENTION_SIZE: 32\n",
            "\tBATCH_NORMALIZATION_MODE: 1\n",
            "\tBATCH_SIZE: 50\n",
            "\tBEAM_SEARCH: True\n",
            "\tBEAM_SIZE: 6\n",
            "\tBETA_1: 0.9\n",
            "\tBETA_2: 0.999\n",
            "\tBIDIRECTIONAL_DEEP_ENCODER: True\n",
            "\tBIDIRECTIONAL_ENCODER: True\n",
            "\tBIDIRECTIONAL_MERGE_MODE: concat\n",
            "\tBPE_CODES_PATH: examples/EuTrans//training_codes.joint\n",
            "\tCLASSIFIER_ACTIVATION: softmax\n",
            "\tCLIP_C: 5.0\n",
            "\tCLIP_V: 0.0\n",
            "\tCOVERAGE_NORM_FACTOR: 0.2\n",
            "\tCOVERAGE_PENALTY: False\n",
            "\tDATASET_NAME: EuTrans\n",
            "\tDATASET_STORE_PATH: datasets/\n",
            "\tDATA_AUGMENTATION: False\n",
            "\tDATA_ROOT_PATH: examples/EuTrans/\n",
            "\tDECODER_HIDDEN_SIZE: 32\n",
            "\tDECODER_RNN_TYPE: ConditionalLSTM\n",
            "\tDEEP_OUTPUT_LAYERS: [('linear', 32)]\n",
            "\tDETOKENIZATION_METHOD: detokenize_none\n",
            "\tDOUBLE_STOCHASTIC_ATTENTION_REG: 0.0\n",
            "\tDROPOUT_P: 0.0\n",
            "\tEARLY_STOP: True\n",
            "\tEMBEDDINGS_FREQ: 1\n",
            "\tENCODER_HIDDEN_SIZE: 32\n",
            "\tENCODER_RNN_TYPE: LSTM\n",
            "\tEPOCHS_FOR_SAVE: 1\n",
            "\tEPSILON: 1e-08\n",
            "\tEVAL_EACH: 1\n",
            "\tEVAL_EACH_EPOCHS: True\n",
            "\tEVAL_ON_SETS: ['val']\n",
            "\tEXTRA_NAME: \n",
            "\tFF_SIZE: 128\n",
            "\tFILL: end\n",
            "\tFORCE_RELOAD_VOCABULARY: False\n",
            "\tGLOSSARY: None\n",
            "\tGRU_RESET_AFTER: True\n",
            "\tHEURISTIC: 0\n",
            "\tHOMOGENEOUS_BATCHES: False\n",
            "\tINIT_ATT: glorot_uniform\n",
            "\tINIT_FUNCTION: glorot_uniform\n",
            "\tINIT_LAYERS: ['tanh']\n",
            "\tINNER_INIT: orthogonal\n",
            "\tINPUTS_IDS_DATASET: ['source_text', 'state_below']\n",
            "\tINPUTS_IDS_MODEL: ['source_text', 'state_below']\n",
            "\tINPUTS_TYPES_DATASET: ['text-features', 'text-features']\n",
            "\tINPUT_VOCABULARY_SIZE: 6880\n",
            "\tJOINT_BATCHES: 4\n",
            "\tKERAS_METRICS: ['perplexity']\n",
            "\tLABEL_SMOOTHING: 0.0\n",
            "\tLENGTH_NORM_FACTOR: 0.2\n",
            "\tLENGTH_PENALTY: False\n",
            "\tLOG_DIR: tensorboard_logs\n",
            "\tLOSS: categorical_crossentropy\n",
            "\tLR: 0.001\n",
            "\tLR_DECAY: None\n",
            "\tLR_GAMMA: 0.8\n",
            "\tLR_HALF_LIFE: 100\n",
            "\tLR_REDUCER_EXP_BASE: -0.5\n",
            "\tLR_REDUCER_TYPE: exponential\n",
            "\tLR_REDUCE_EACH_EPOCHS: False\n",
            "\tLR_START_REDUCTION_ON_EPOCH: 0\n",
            "\tMAPPING: examples/EuTrans//mapping.es_en.pkl\n",
            "\tMAXLEN_GIVEN_X: True\n",
            "\tMAXLEN_GIVEN_X_FACTOR: 2\n",
            "\tMAX_EPOCH: 500\n",
            "\tMAX_INPUT_TEXT_LEN: 50\n",
            "\tMAX_OUTPUT_TEXT_LEN: 50\n",
            "\tMAX_OUTPUT_TEXT_LEN_TEST: 150\n",
            "\tMAX_PLOT_Y: 100.0\n",
            "\tMETRICS: ['sacrebleu', 'perplexity']\n",
            "\tMINLEN_GIVEN_X: True\n",
            "\tMINLEN_GIVEN_X_FACTOR: 3\n",
            "\tMIN_DELTA: 0.0\n",
            "\tMIN_LR: 1e-09\n",
            "\tMIN_OCCURRENCES_INPUT_VOCAB: 0\n",
            "\tMIN_OCCURRENCES_OUTPUT_VOCAB: 0\n",
            "\tMODE: training\n",
            "\tMODEL_NAME: EuTrans_esen_AttentionRNNEncoderDecoder_src_emb_32_bidir_True_enc_LSTM_32_dec_ConditionalLSTM_32_deepout_linear_trg_emb_32_Adam_0.001\n",
            "\tMODEL_SIZE: 32\n",
            "\tMODEL_TYPE: AttentionRNNEncoderDecoder\n",
            "\tMOMENTUM: 0.0\n",
            "\tMULTIHEAD_ATTENTION_ACTIVATION: linear\n",
            "\tNESTEROV_MOMENTUM: False\n",
            "\tNOISE_AMOUNT: 0.01\n",
            "\tNORMALIZE_SAMPLING: False\n",
            "\tN_GPUS: 1\n",
            "\tN_HEADS: 8\n",
            "\tN_LAYERS_DECODER: 1\n",
            "\tN_LAYERS_ENCODER: 1\n",
            "\tN_SAMPLES: 5\n",
            "\tOPTIMIZED_SEARCH: True\n",
            "\tOPTIMIZER: Adam\n",
            "\tOUTPUTS_IDS_DATASET: ['target_text']\n",
            "\tOUTPUTS_IDS_MODEL: ['target_text']\n",
            "\tOUTPUTS_TYPES_DATASET: ['text-features']\n",
            "\tOUTPUT_VOCABULARY_SIZE: 6287\n",
            "\tPAD_ON_BATCH: True\n",
            "\tPARALLEL_LOADERS: 1\n",
            "\tPATIENCE: 10\n",
            "\tPLOT_EVALUATION: False\n",
            "\tPOS_UNK: True\n",
            "\tREBUILD_DATASET: True\n",
            "\tRECURRENT_DROPOUT_P: 0.0\n",
            "\tRECURRENT_INPUT_DROPOUT_P: 0.0\n",
            "\tRECURRENT_WEIGHT_DECAY: 0.0\n",
            "\tREGULARIZATION_FN: L2\n",
            "\tRELOAD: 0\n",
            "\tRELOAD_EPOCH: True\n",
            "\tRHO: 0.9\n",
            "\tSAMPLE_EACH_UPDATES: 300\n",
            "\tSAMPLE_ON_SETS: ['train', 'val']\n",
            "\tSAMPLE_WEIGHTS: True\n",
            "\tSAMPLING: max_likelihood\n",
            "\tSAMPLING_SAVE_MODE: list\n",
            "\tSAVE_EACH_EVALUATION: True\n",
            "\tSCALE_SOURCE_WORD_EMBEDDINGS: False\n",
            "\tSCALE_TARGET_WORD_EMBEDDINGS: False\n",
            "\tSEARCH_PRUNING: False\n",
            "\tSKIP_VECTORS_HIDDEN_SIZE: 32\n",
            "\tSKIP_VECTORS_SHARED_ACTIVATION: tanh\n",
            "\tSOURCE_TEXT_EMBEDDING_SIZE: 32\n",
            "\tSRC_LAN: es\n",
            "\tSRC_PRETRAINED_VECTORS: None\n",
            "\tSRC_PRETRAINED_VECTORS_TRAINABLE: True\n",
            "\tSTART_EVAL_ON_EPOCH: 1\n",
            "\tSTART_SAMPLING_ON_EPOCH: 1\n",
            "\tSTOP_METRIC: Bleu_4\n",
            "\tSTORE_PATH: trained_models/EuTrans_esen_AttentionRNNEncoderDecoder_src_emb_32_bidir_True_enc_LSTM_32_dec_ConditionalLSTM_32_deepout_linear_trg_emb_32_Adam_0.001/\n",
            "\tTARGET_TEXT_EMBEDDING_SIZE: 32\n",
            "\tTASK_NAME: EuTrans\n",
            "\tTEMPERATURE: 1\n",
            "\tTENSORBOARD: True\n",
            "\tTEXT_FILES: {'train': 'training.', 'val': 'dev.', 'test': 'test.'}\n",
            "\tTIE_EMBEDDINGS: False\n",
            "\tTOKENIZATION_METHOD: tokenize_none\n",
            "\tTOKENIZE_HYPOTHESES: True\n",
            "\tTOKENIZE_REFERENCES: True\n",
            "\tTRAINABLE_DECODER: True\n",
            "\tTRAINABLE_ENCODER: True\n",
            "\tTRAIN_ON_TRAINVAL: False\n",
            "\tTRG_LAN: en\n",
            "\tTRG_PRETRAINED_VECTORS: None\n",
            "\tTRG_PRETRAINED_VECTORS_TRAINABLE: True\n",
            "\tUSE_BATCH_NORMALIZATION: True\n",
            "\tUSE_CUDNN: False\n",
            "\tUSE_L1: False\n",
            "\tUSE_L2: False\n",
            "\tUSE_NOISE: False\n",
            "\tUSE_PRELU: False\n",
            "\tUSE_TF_OPTIMIZER: True\n",
            "\tVERBOSE: 1\n",
            "\tWARMUP_EXP: -1.5\n",
            "\tWEIGHT_DECAY: 0.0001\n",
            "\tWRITE_VALID_SAMPLES: True\n",
            "-----------------------------------------------------------------------------------\n",
            "Model: \"tutorial_model_training\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "source_text (InputLayer)        (None, None)         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "source_word_embedding (Embeddin (None, None, 32)     220160      source_text[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "src_embedding_batch_normalizati (None, None, 32)     128         source_word_embedding[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "remove_mask_1 (RemoveMask)      (None, None, 32)     0           src_embedding_batch_normalization\n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_encoder_LSTM (Bid (None, None, 64)     16640       remove_mask_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "annotations_batch_normalization (None, None, 64)     256         bidirectional_encoder_LSTM[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "source_text_mask (GetMask)      (None, None, 32)     0           src_embedding_batch_normalization\n",
            "__________________________________________________________________________________________________\n",
            "annotations (ApplyMask)         (None, None, 64)     0           annotations_batch_normalization[0\n",
            "                                                                 source_text_mask[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "state_below (InputLayer)        (None, None)         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "ctx_mean (MaskedMean)           (None, 64)           0           annotations[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "target_word_embedding (Embeddin (None, None, 32)     201184      state_below[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "initial_state (Dense)           (None, 32)           2080        ctx_mean[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "initial_memory (Dense)          (None, 32)           2080        ctx_mean[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "state_below_batch_normalization (None, None, 32)     128         target_word_embedding[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "initial_state_batch_normalizati (None, 32)           128         initial_state[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "initial_memory_batch_normalizat (None, 32)           128         initial_memory[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "decoder_AttConditionalLSTMCond  [(None, None, 32), ( 23873       state_below_batch_normalization[0\n",
            "                                                                 annotations[0][0]                \n",
            "                                                                 initial_state_batch_normalization\n",
            "                                                                 initial_memory_batch_normalizatio\n",
            "__________________________________________________________________________________________________\n",
            "proj_h0_batch_normalization (Ba (None, None, 32)     128         decoder_AttConditionalLSTMCond[0]\n",
            "__________________________________________________________________________________________________\n",
            "logit_ctx (TimeDistributed)     (None, None, 32)     2080        decoder_AttConditionalLSTMCond[0]\n",
            "__________________________________________________________________________________________________\n",
            "logit_lstm (TimeDistributed)    (None, None, 32)     1056        proj_h0_batch_normalization[0][0]\n",
            "__________________________________________________________________________________________________\n",
            "permute_general_1 (PermuteGener (None, None, 32)     0           logit_ctx[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "logit_emb (TimeDistributed)     (None, None, 32)     1056        state_below_batch_normalization[0\n",
            "__________________________________________________________________________________________________\n",
            "out_layer_mlp_batch_normalizati (None, None, 32)     128         logit_lstm[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "out_layer_ctx_batch_normalizati (None, None, 32)     128         permute_general_1[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "out_layer_emb_batch_normalizati (None, None, 32)     128         logit_emb[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "additional_input (Add)          (None, None, 32)     0           out_layer_mlp_batch_normalization\n",
            "                                                                 out_layer_ctx_batch_normalization\n",
            "                                                                 out_layer_emb_batch_normalization\n",
            "__________________________________________________________________________________________________\n",
            "activation_1 (Activation)       (None, None, 32)     0           additional_input[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "linear_0 (TimeDistributed)      (None, None, 32)     1056        activation_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "out_layer_linear_0_batch_normal (None, None, 32)     128         linear_0[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "target_text (TimeDistributed)   (None, None, 6287)   207471      out_layer_linear_0_batch_normaliz\n",
            "==================================================================================================\n",
            "Total params: 680,144\n",
            "Trainable params: 679,440\n",
            "Non-trainable params: 704\n",
            "__________________________________________________________________________________________________\n",
            "WARNING:tensorflow:From /content/drive/My Drive/Urdu2/nmt-keras/nmt_keras/model_zoo.py:213: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:15] From /content/drive/My Drive/Urdu2/nmt-keras/nmt_keras/model_zoo.py:213: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n",
            "[23/06/2020 17:24:15] Preparing optimizer and compiling. Optimizer configuration: \n",
            "\t LR: 0.001\n",
            "\t LOSS: categorical_crossentropy\n",
            "\t BETA_1: 0.9\n",
            "\t BETA_2: 0.999\n",
            "\t EPSILON: 1e-08\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1192: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 17:24:15] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1192: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STUTv8cvNXTw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "inputMapping = dict()\n",
        "for i, id_in in enumerate(params['INPUTS_IDS_DATASET']):\n",
        "    pos_source = dataset.ids_inputs.index(id_in)\n",
        "    id_dest = nmt_model.ids_inputs[i]\n",
        "    inputMapping[id_dest] = pos_source\n",
        "nmt_model.setInputsMapping(inputMapping)\n",
        "\n",
        "outputMapping = dict()\n",
        "for i, id_out in enumerate(params['OUTPUTS_IDS_DATASET']):\n",
        "    pos_target = dataset.ids_outputs.index(id_out)\n",
        "    id_dest = nmt_model.ids_outputs[i]\n",
        "    outputMapping[id_dest] = pos_target\n",
        "nmt_model.setOutputsMapping(outputMapping)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YL_Whu85NqbO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "is_transformer = params.get('ATTEND_ON_OUTPUT', 'transformer' in params['MODEL_TYPE'].lower())\n",
        "search_params = {\n",
        "    'language': 'en',\n",
        "    'tokenize_f': eval('dataset.' + 'tokenize_none'),\n",
        "    'beam_size': 12,\n",
        "    'optimized_search': True,\n",
        "    'model_inputs': params['INPUTS_IDS_MODEL'],\n",
        "    'model_outputs': params['OUTPUTS_IDS_MODEL'],\n",
        "    'dataset_inputs':  params['INPUTS_IDS_DATASET'],\n",
        "    'dataset_outputs':  params['OUTPUTS_IDS_DATASET'],\n",
        "    'n_parallel_loaders': 1,\n",
        "    'maxlen': 50,\n",
        "    'normalize_probs': True,\n",
        "    'pos_unk': True and not is_transformer,  # Pos_unk is unimplemented for transformer models\n",
        "    'heuristic': 0,\n",
        "    'state_below_maxlen': -1,\n",
        "    'attend_on_output': is_transformer,\n",
        "    'val': {'references': dataset.extra_variables['val']['target_text']}\n",
        "  }\n",
        "\n",
        "vocab = dataset.vocabulary['target_text']['idx2words']\n",
        "callbacks = []\n",
        "input_text_id = params['INPUTS_IDS_DATASET'][0]\n",
        "\n",
        "callbacks.append(PrintPerformanceMetricOnEpochEndOrEachNUpdates(nmt_model,\n",
        "                                                                dataset,\n",
        "                                                                gt_id='target_text',\n",
        "                                                                metric_name=['sacrebleu'],\n",
        "                                                                set_name=['val'],\n",
        "                                                                batch_size=50,\n",
        "                                                                each_n_epochs=1,\n",
        "                                                                extra_vars=search_params,\n",
        "                                                                reload_epoch=0,\n",
        "                                                                is_text=True,\n",
        "                                                                input_text_id=input_text_id,\n",
        "                                                                index2word_y=vocab,\n",
        "                                                                sampling_type='max_likelihood',\n",
        "                                                                beam_search=True,\n",
        "                                                                save_path=nmt_model.model_path,\n",
        "                                                                start_eval_on_epoch=0,\n",
        "                                                                write_samples=True,\n",
        "                                                                write_type='list',\n",
        "                                                                verbose=True))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mk6nuynNv-o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "training_params = {'n_epochs': 100,\n",
        "                   'batch_size': 50,\n",
        "                   'maxlen': 50,\n",
        "                   'epochs_for_save': 1,\n",
        "                   'verbose': 1,\n",
        "                   'eval_on_sets': [], \n",
        "                   'n_parallel_loaders': 1,\n",
        "                   'extra_callbacks': callbacks,\n",
        "                   'reload_epoch': 0,\n",
        "                   'epoch_offset': 0}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pnKFUJ9XN3Ef",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "45b8a0e2-91d4-4713-eb23-9d40a4a8a01c"
      },
      "source": [
        "nmt_model.trainNet(dataset, training_params)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:11:23] <<< Training model >>>\n",
            "[23/06/2020 18:11:23] Training parameters: { \n",
            "\tbatch_size: 50\n",
            "\tclass_weights: None\n",
            "\tda_enhance_list: []\n",
            "\tda_patch_type: resize_and_rndcrop\n",
            "\tdata_augmentation: False\n",
            "\teach_n_epochs: 1\n",
            "\tepoch_offset: 0\n",
            "\tepochs_for_save: 1\n",
            "\teval_on_epochs: True\n",
            "\teval_on_sets: []\n",
            "\textra_callbacks: [<keras_wrapper.extra.callbacks.EvalPerformance object at 0x7fb0981e93c8>]\n",
            "\thomogeneous_batches: False\n",
            "\tinitial_lr: 1.0\n",
            "\tjoint_batches: 4\n",
            "\tlr_decay: None\n",
            "\tlr_gamma: 0.1\n",
            "\tlr_half_life: 50000\n",
            "\tlr_reducer_exp_base: 0.5\n",
            "\tlr_reducer_type: linear\n",
            "\tlr_warmup_exp: -1.5\n",
            "\tmaxlen: 50\n",
            "\tmean_substraction: False\n",
            "\tmetric_check: None\n",
            "\tmin_delta: 0.0\n",
            "\tmin_lr: 1e-09\n",
            "\tn_epochs: 100\n",
            "\tn_gpus: 1\n",
            "\tn_parallel_loaders: 1\n",
            "\tnormalization_type: None\n",
            "\tnormalize: False\n",
            "\tnum_iterations_val: None\n",
            "\tpatience: 0\n",
            "\tpatience_check_split: val\n",
            "\treduce_each_epochs: True\n",
            "\treload_epoch: 0\n",
            "\tshuffle: True\n",
            "\tstart_eval_on_epoch: 0\n",
            "\tstart_reduction_on_epoch: 0\n",
            "\ttensorboard: False\n",
            "\ttensorboard_params: {'log_dir': 'tensorboard_logs', 'histogram_freq': 0, 'batch_size': 50, 'write_graph': True, 'write_grads': False, 'write_images': False, 'embeddings_freq': 0, 'embeddings_layer_names': None, 'embeddings_metadata': None, 'update_freq': 'epoch'}\n",
            "\tverbose: 1\n",
            "\two_da_patch_type: whole\n",
            "}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "148/148 [==============================] - 99s 669ms/step - loss: 1.6462 - perplexity: 1611.0706\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:13:02] <<< Saving model to trained_models/tutorial_model/epoch_1 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py:165: UserWarning: TensorFlow optimizers do not make it possible to access optimizer attributes or optimizer state after instantiation. As a result, we cannot save the optimizer as part of the model save file.You will have to compile your model again after loading it. Prefer using a Keras optimizer instead (see keras.io/optimizers).\n",
            "  'TensorFlow optimizers do not '\n",
            "[23/06/2020 18:13:05] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:13:05] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 11879.292215 \t Average cost: 39.597641\n",
            "The sampling took: 58.747998 secs (Speed: 0.195827 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:14:04] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:14:04] Decoding beam search prediction ...\n",
            "[23/06/2020 18:14:04] Using heuristic 0\n",
            "[23/06/2020 18:14:04] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:14:04] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:14:04] Bleu_4: 7.009974628993629\n",
            "[23/06/2020 18:14:04] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:14:04] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 2/100\n",
            "148/148 [==============================] - 98s 665ms/step - loss: 1.6226 - perplexity: 1682.6012\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:15:42] <<< Saving model to trained_models/tutorial_model/epoch_2 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:15:46] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:15:46] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 11944.697590 \t Average cost: 39.815659\n",
            "The sampling took: 59.577899 secs (Speed: 0.198593 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:16:46] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:16:46] Decoding beam search prediction ...\n",
            "[23/06/2020 18:16:46] Using heuristic 0\n",
            "[23/06/2020 18:16:46] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:16:46] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:16:46] Bleu_4: 6.894715725773503\n",
            "[23/06/2020 18:16:46] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:16:46] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 3/100\n",
            "148/148 [==============================] - 98s 660ms/step - loss: 1.5890 - perplexity: 1637.3665\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:18:24] <<< Saving model to trained_models/tutorial_model/epoch_3 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:18:27] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:18:27] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 11160.922287 \t Average cost: 37.203074\n",
            "The sampling took: 56.329079 secs (Speed: 0.187764 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:19:23] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:19:23] Decoding beam search prediction ...\n",
            "[23/06/2020 18:19:23] Using heuristic 0\n",
            "[23/06/2020 18:19:23] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:19:24] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:19:24] Bleu_4: 7.58360665881682\n",
            "[23/06/2020 18:19:24] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:19:24] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 4/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.5650 - perplexity: 1576.6515\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:21:02] <<< Saving model to trained_models/tutorial_model/epoch_4 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:21:05] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:21:05] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 11093.607568 \t Average cost: 36.978692\n",
            "The sampling took: 56.080280 secs (Speed: 0.186934 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:22:01] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:22:01] Decoding beam search prediction ...\n",
            "[23/06/2020 18:22:01] Using heuristic 0\n",
            "[23/06/2020 18:22:01] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:22:02] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:22:02] Bleu_4: 7.825421278184139\n",
            "[23/06/2020 18:22:02] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:22:02] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 5/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.5292 - perplexity: 1635.9790\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:23:39] <<< Saving model to trained_models/tutorial_model/epoch_5 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:23:42] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:23:42] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10908.095101 \t Average cost: 36.360317\n",
            "The sampling took: 56.821980 secs (Speed: 0.189407 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:24:39] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:24:39] Decoding beam search prediction ...\n",
            "[23/06/2020 18:24:39] Using heuristic 0\n",
            "[23/06/2020 18:24:39] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:24:39] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:24:39] Bleu_4: 7.800386738428815\n",
            "[23/06/2020 18:24:39] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:24:39] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 6/100\n",
            "148/148 [==============================] - 96s 652ms/step - loss: 1.5097 - perplexity: 1597.5454\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:26:16] <<< Saving model to trained_models/tutorial_model/epoch_6 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:26:19] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:26:19] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9949.982662 \t Average cost: 33.166609\n",
            "The sampling took: 55.191731 secs (Speed: 0.183972 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:27:15] Prediction output 0: target_text (text)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:27:15] Decoding beam search prediction ...\n",
            "[23/06/2020 18:27:15] Using heuristic 0\n",
            "[23/06/2020 18:27:15] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:27:15] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:27:15] Bleu_4: 7.629971176392251\n",
            "[23/06/2020 18:27:15] Done evaluating on metric sacrebleu\n",
            "[23/06/2020 18:27:15] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 7/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.4854 - perplexity: 1701.8551\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:28:51] <<< Saving model to trained_models/tutorial_model/epoch_7 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:28:55] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:28:55] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10467.726643 \t Average cost: 34.892422\n",
            "The sampling took: 57.004494 secs (Speed: 0.190015 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:29:52] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:29:52] Decoding beam search prediction ...\n",
            "[23/06/2020 18:29:52] Using heuristic 0\n",
            "[23/06/2020 18:29:52] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:29:52] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:29:52] Bleu_4: 8.27149507694913\n",
            "[23/06/2020 18:29:52] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:29:52] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 8/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.4739 - perplexity: 1597.7299\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:31:28] <<< Saving model to trained_models/tutorial_model/epoch_8 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:31:32] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:31:32] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9827.814249 \t Average cost: 32.759381\n",
            "The sampling took: 54.918010 secs (Speed: 0.183060 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:32:27] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:32:27] Decoding beam search prediction ...\n",
            "[23/06/2020 18:32:27] Using heuristic 0\n",
            "[23/06/2020 18:32:27] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:32:27] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:32:27] Bleu_4: 8.816133327241486\n",
            "[23/06/2020 18:32:27] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:32:27] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 9/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.4478 - perplexity: 1604.8678\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:34:03] <<< Saving model to trained_models/tutorial_model/epoch_9 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:34:07] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:34:07] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10416.991266 \t Average cost: 34.723304\n",
            "The sampling took: 56.905189 secs (Speed: 0.189684 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:35:04] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:35:04] Decoding beam search prediction ...\n",
            "[23/06/2020 18:35:04] Using heuristic 0\n",
            "[23/06/2020 18:35:04] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:35:04] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:35:04] Bleu_4: 7.736589113006643\n",
            "[23/06/2020 18:35:04] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:35:04] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 10/100\n",
            "148/148 [==============================] - 97s 654ms/step - loss: 1.4236 - perplexity: 1646.2047\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:36:41] <<< Saving model to trained_models/tutorial_model/epoch_10 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:36:44] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:36:44] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10267.964418 \t Average cost: 34.226548\n",
            "The sampling took: 57.787894 secs (Speed: 0.192626 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:37:42] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:37:42] Decoding beam search prediction ...\n",
            "[23/06/2020 18:37:42] Using heuristic 0\n",
            "[23/06/2020 18:37:42] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:37:42] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:37:42] Bleu_4: 8.304571166204102\n",
            "[23/06/2020 18:37:42] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:37:42] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 11/100\n",
            "148/148 [==============================] - 99s 668ms/step - loss: 1.3974 - perplexity: 1747.8503\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:39:21] <<< Saving model to trained_models/tutorial_model/epoch_11 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:39:24] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:39:24] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10071.940767 \t Average cost: 33.573136\n",
            "The sampling took: 57.239771 secs (Speed: 0.190799 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:40:22] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:40:22] Decoding beam search prediction ...\n",
            "[23/06/2020 18:40:22] Using heuristic 0\n",
            "[23/06/2020 18:40:22] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:40:22] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:40:22] Bleu_4: 8.232427311659645\n",
            "[23/06/2020 18:40:22] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:40:22] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 12/100\n",
            "148/148 [==============================] - 98s 665ms/step - loss: 1.3900 - perplexity: 1875.0613\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:42:00] <<< Saving model to trained_models/tutorial_model/epoch_12 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:42:04] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:42:04] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9355.570773 \t Average cost: 31.185236\n",
            "The sampling took: 57.196407 secs (Speed: 0.190655 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:43:01] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:43:01] Decoding beam search prediction ...\n",
            "[23/06/2020 18:43:01] Using heuristic 0\n",
            "[23/06/2020 18:43:01] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:43:01] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:43:01] Bleu_4: 8.899988692684992\n",
            "[23/06/2020 18:43:01] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:43:01] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 13/100\n",
            "148/148 [==============================] - 98s 665ms/step - loss: 1.3808 - perplexity: 1729.6899\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:44:40] <<< Saving model to trained_models/tutorial_model/epoch_13 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:44:43] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:44:43] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 10186.382034 \t Average cost: 33.954607\n",
            "The sampling took: 58.404891 secs (Speed: 0.194683 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:45:42] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:45:42] Decoding beam search prediction ...\n",
            "[23/06/2020 18:45:42] Using heuristic 0\n",
            "[23/06/2020 18:45:42] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:45:42] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:45:42] Bleu_4: 8.55623259950389\n",
            "[23/06/2020 18:45:42] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:45:42] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 14/100\n",
            "148/148 [==============================] - 99s 670ms/step - loss: 1.3565 - perplexity: 1836.0310\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:47:21] <<< Saving model to trained_models/tutorial_model/epoch_14 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:47:25] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:47:25] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9478.671636 \t Average cost: 31.595572\n",
            "The sampling took: 56.416391 secs (Speed: 0.188055 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:48:21] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:48:21] Decoding beam search prediction ...\n",
            "[23/06/2020 18:48:21] Using heuristic 0\n",
            "[23/06/2020 18:48:21] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:48:21] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:48:21] Bleu_4: 8.652361272841047\n",
            "[23/06/2020 18:48:21] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:48:21] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 15/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.3416 - perplexity: 1783.3937\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:49:58] <<< Saving model to trained_models/tutorial_model/epoch_15 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:50:01] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:50:01] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9175.879238 \t Average cost: 30.586264\n",
            "The sampling took: 56.107701 secs (Speed: 0.187026 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:50:57] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:50:57] Decoding beam search prediction ...\n",
            "[23/06/2020 18:50:57] Using heuristic 0\n",
            "[23/06/2020 18:50:57] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:50:57] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:50:57] Bleu_4: 8.976471633454873\n",
            "[23/06/2020 18:50:57] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:50:57] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_15.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 16/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.3274 - perplexity: 1874.9076\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:52:35] <<< Saving model to trained_models/tutorial_model/epoch_16 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:52:35] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:52:35] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9900.697315 \t Average cost: 33.002324\n",
            "The sampling took: 58.376567 secs (Speed: 0.194589 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:53:33] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:53:33] Decoding beam search prediction ...\n",
            "[23/06/2020 18:53:33] Using heuristic 0\n",
            "[23/06/2020 18:53:33] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:53:34] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:53:34] Bleu_4: 8.7834804015841\n",
            "[23/06/2020 18:53:34] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:53:34] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_16.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 17/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.3147 - perplexity: 1766.5022\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:55:11] <<< Saving model to trained_models/tutorial_model/epoch_17 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:55:11] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:55:11] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8859.032741 \t Average cost: 29.530109\n",
            "The sampling took: 56.893563 secs (Speed: 0.189645 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:56:08] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:56:08] Decoding beam search prediction ...\n",
            "[23/06/2020 18:56:08] Using heuristic 0\n",
            "[23/06/2020 18:56:08] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:56:08] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:56:08] Bleu_4: 8.516876975254078\n",
            "[23/06/2020 18:56:08] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:56:08] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_17.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 18/100\n",
            "148/148 [==============================] - 98s 659ms/step - loss: 1.3025 - perplexity: 2008.4207\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:57:46] <<< Saving model to trained_models/tutorial_model/epoch_18 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:57:46] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 18:57:46] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9028.487004 \t Average cost: 30.094957\n",
            "The sampling took: 57.937712 secs (Speed: 0.193126 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:58:44] Prediction output 0: target_text (text)\n",
            "[23/06/2020 18:58:44] Decoding beam search prediction ...\n",
            "[23/06/2020 18:58:44] Using heuristic 0\n",
            "[23/06/2020 18:58:44] Evaluating on metric sacrebleu\n",
            "[23/06/2020 18:58:44] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 18:58:44] Bleu_4: 8.497198636353883\n",
            "[23/06/2020 18:58:44] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 18:58:44] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_18.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 19/100\n",
            "148/148 [==============================] - 97s 654ms/step - loss: 1.2897 - perplexity: 1883.3190\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:00:21] <<< Saving model to trained_models/tutorial_model/epoch_19 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:00:22] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:00:22] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9046.532516 \t Average cost: 30.155108\n",
            "The sampling took: 56.822181 secs (Speed: 0.189407 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:01:18] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:01:18] Decoding beam search prediction ...\n",
            "[23/06/2020 19:01:18] Using heuristic 0\n",
            "[23/06/2020 19:01:18] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:01:19] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:01:19] Bleu_4: 8.716377088962053\n",
            "[23/06/2020 19:01:19] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:01:19] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_19.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 20/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.2740 - perplexity: 1993.1250\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:02:56] <<< Saving model to trained_models/tutorial_model/epoch_20 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:02:56] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:02:56] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 9245.900350 \t Average cost: 30.819668\n",
            "The sampling took: 57.989782 secs (Speed: 0.193299 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:03:54] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:03:54] Decoding beam search prediction ...\n",
            "[23/06/2020 19:03:54] Using heuristic 0\n",
            "[23/06/2020 19:03:54] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:03:54] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:03:54] Bleu_4: 8.875046145578413\n",
            "[23/06/2020 19:03:54] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:03:54] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_20.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 21/100\n",
            "148/148 [==============================] - 97s 658ms/step - loss: 1.2693 - perplexity: 1888.9354\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:05:32] <<< Saving model to trained_models/tutorial_model/epoch_21 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:05:32] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:05:32] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8940.594786 \t Average cost: 29.801983\n",
            "The sampling took: 56.610059 secs (Speed: 0.188700 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:06:29] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:06:29] Decoding beam search prediction ...\n",
            "[23/06/2020 19:06:29] Using heuristic 0\n",
            "[23/06/2020 19:06:29] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:06:29] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:06:29] Bleu_4: 8.739371569135974\n",
            "[23/06/2020 19:06:29] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:06:29] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_21.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 22/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.2618 - perplexity: 1973.5621\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:08:06] <<< Saving model to trained_models/tutorial_model/epoch_22 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:08:06] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:08:06] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8510.512090 \t Average cost: 28.368374\n",
            "The sampling took: 55.259670 secs (Speed: 0.184199 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:09:01] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:09:01] Decoding beam search prediction ...\n",
            "[23/06/2020 19:09:01] Using heuristic 0\n",
            "[23/06/2020 19:09:01] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:09:01] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:09:01] Bleu_4: 8.809287646944238\n",
            "[23/06/2020 19:09:01] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:09:01] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_22.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 23/100\n",
            "148/148 [==============================] - 97s 658ms/step - loss: 1.2431 - perplexity: 1884.2184\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:10:39] <<< Saving model to trained_models/tutorial_model/epoch_23 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:10:39] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:10:39] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8821.813992 \t Average cost: 29.406047\n",
            "The sampling took: 56.268428 secs (Speed: 0.187561 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:11:35] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:11:35] Decoding beam search prediction ...\n",
            "[23/06/2020 19:11:35] Using heuristic 0\n",
            "[23/06/2020 19:11:35] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:11:36] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:11:36] Bleu_4: 8.685611427236847\n",
            "[23/06/2020 19:11:36] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:11:36] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_23.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 24/100\n",
            "148/148 [==============================] - 97s 658ms/step - loss: 1.2310 - perplexity: 2045.6412\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:13:13] <<< Saving model to trained_models/tutorial_model/epoch_24 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:13:13] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:13:13] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8332.341092 \t Average cost: 27.774470\n",
            "The sampling took: 55.753481 secs (Speed: 0.185845 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:14:09] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:14:09] Decoding beam search prediction ...\n",
            "[23/06/2020 19:14:09] Using heuristic 0\n",
            "[23/06/2020 19:14:09] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:14:09] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:14:09] Bleu_4: 8.843577226416436\n",
            "[23/06/2020 19:14:09] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:14:09] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_24.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 25/100\n",
            "148/148 [==============================] - 98s 659ms/step - loss: 1.2244 - perplexity: 2139.3303\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:15:47] <<< Saving model to trained_models/tutorial_model/epoch_25 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:15:47] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:15:47] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8281.645964 \t Average cost: 27.605487\n",
            "The sampling took: 56.084426 secs (Speed: 0.186948 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:16:43] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:16:43] Decoding beam search prediction ...\n",
            "[23/06/2020 19:16:43] Using heuristic 0\n",
            "[23/06/2020 19:16:43] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:16:43] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:16:43] Bleu_4: 8.646378782651166\n",
            "[23/06/2020 19:16:43] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:16:44] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_25.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 26/100\n",
            "148/148 [==============================] - 97s 658ms/step - loss: 1.2115 - perplexity: 2139.6443\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:18:21] <<< Saving model to trained_models/tutorial_model/epoch_26 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:18:21] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:18:21] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8404.309998 \t Average cost: 28.014367\n",
            "The sampling took: 56.483689 secs (Speed: 0.188279 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:19:18] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:19:18] Decoding beam search prediction ...\n",
            "[23/06/2020 19:19:18] Using heuristic 0\n",
            "[23/06/2020 19:19:18] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:19:18] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:19:18] Bleu_4: 9.418496627483602\n",
            "[23/06/2020 19:19:18] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:19:18] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_26.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 27/100\n",
            "148/148 [==============================] - 98s 660ms/step - loss: 1.2094 - perplexity: 2374.4663\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:20:56] <<< Saving model to trained_models/tutorial_model/epoch_27 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:20:56] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:20:56] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8298.181802 \t Average cost: 27.660606\n",
            "The sampling took: 55.600189 secs (Speed: 0.185334 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:21:52] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:21:52] Decoding beam search prediction ...\n",
            "[23/06/2020 19:21:52] Using heuristic 0\n",
            "[23/06/2020 19:21:52] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:21:52] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:21:52] Bleu_4: 9.095676923152608\n",
            "[23/06/2020 19:21:52] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:21:52] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_27.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 28/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.2092 - perplexity: 2203.2466\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:23:28] <<< Saving model to trained_models/tutorial_model/epoch_28 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:23:29] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:23:29] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7880.549627 \t Average cost: 26.268499\n",
            "The sampling took: 57.004406 secs (Speed: 0.190015 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:24:26] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:24:26] Decoding beam search prediction ...\n",
            "[23/06/2020 19:24:26] Using heuristic 0\n",
            "[23/06/2020 19:24:26] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:24:26] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:24:26] Bleu_4: 9.474079631277391\n",
            "[23/06/2020 19:24:26] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:24:26] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_28.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 29/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.1896 - perplexity: 2179.9573\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:26:03] <<< Saving model to trained_models/tutorial_model/epoch_29 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:26:03] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:26:03] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 8625.219885 \t Average cost: 28.750733\n",
            "The sampling took: 57.541613 secs (Speed: 0.191805 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:27:01] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:27:01] Decoding beam search prediction ...\n",
            "[23/06/2020 19:27:01] Using heuristic 0\n",
            "[23/06/2020 19:27:01] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:27:01] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:27:01] Bleu_4: 9.209539760605637\n",
            "[23/06/2020 19:27:01] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:27:01] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_29.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 30/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.1777 - perplexity: 2387.1628\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:28:37] <<< Saving model to trained_models/tutorial_model/epoch_30 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:28:38] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:28:38] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7844.249451 \t Average cost: 26.147498\n",
            "The sampling took: 56.419145 secs (Speed: 0.188064 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:29:34] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:29:34] Decoding beam search prediction ...\n",
            "[23/06/2020 19:29:34] Using heuristic 0\n",
            "[23/06/2020 19:29:34] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:29:34] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:29:34] Bleu_4: 9.335717735079484\n",
            "[23/06/2020 19:29:34] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:29:35] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_30.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 31/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.1672 - perplexity: 2311.8721\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:31:11] <<< Saving model to trained_models/tutorial_model/epoch_31 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:31:12] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:31:12] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7421.389487 \t Average cost: 24.737965\n",
            "The sampling took: 55.032388 secs (Speed: 0.183441 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:32:07] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:32:07] Decoding beam search prediction ...\n",
            "[23/06/2020 19:32:07] Using heuristic 0\n",
            "[23/06/2020 19:32:07] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:32:07] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:32:07] Bleu_4: 9.105258941318613\n",
            "[23/06/2020 19:32:07] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:32:07] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_31.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 32/100\n",
            "148/148 [==============================] - 96s 650ms/step - loss: 1.1679 - perplexity: 2411.9858\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:33:43] <<< Saving model to trained_models/tutorial_model/epoch_32 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:33:43] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:33:43] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7849.244230 \t Average cost: 26.164147\n",
            "The sampling took: 57.883200 secs (Speed: 0.192944 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:34:41] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:34:41] Decoding beam search prediction ...\n",
            "[23/06/2020 19:34:41] Using heuristic 0\n",
            "[23/06/2020 19:34:41] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:34:41] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:34:41] Bleu_4: 8.953492701067121\n",
            "[23/06/2020 19:34:41] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:34:42] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_32.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 33/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.1637 - perplexity: 2437.5657\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:36:19] <<< Saving model to trained_models/tutorial_model/epoch_33 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:36:20] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:36:20] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7493.437521 \t Average cost: 24.978125\n",
            "The sampling took: 56.270091 secs (Speed: 0.187567 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:37:16] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:37:16] Decoding beam search prediction ...\n",
            "[23/06/2020 19:37:16] Using heuristic 0\n",
            "[23/06/2020 19:37:16] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:37:16] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:37:16] Bleu_4: 10.110711630817915\n",
            "[23/06/2020 19:37:16] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:37:16] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_33.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 34/100\n",
            "148/148 [==============================] - 97s 656ms/step - loss: 1.1581 - perplexity: 2330.5078\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:38:53] <<< Saving model to trained_models/tutorial_model/epoch_34 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:38:54] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:38:54] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7428.489572 \t Average cost: 24.761632\n",
            "The sampling took: 55.773185 secs (Speed: 0.185911 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:39:50] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:39:50] Decoding beam search prediction ...\n",
            "[23/06/2020 19:39:50] Using heuristic 0\n",
            "[23/06/2020 19:39:50] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:39:50] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:39:50] Bleu_4: 9.292076443913254\n",
            "[23/06/2020 19:39:50] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:39:50] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_34.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 35/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.1535 - perplexity: 2338.4565\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:41:26] <<< Saving model to trained_models/tutorial_model/epoch_35 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:41:26] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:41:26] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7607.971143 \t Average cost: 25.359904\n",
            "The sampling took: 53.998485 secs (Speed: 0.179995 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:42:20] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:42:20] Decoding beam search prediction ...\n",
            "[23/06/2020 19:42:20] Using heuristic 0\n",
            "[23/06/2020 19:42:20] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:42:21] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:42:21] Bleu_4: 9.513842040611186\n",
            "[23/06/2020 19:42:21] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:42:21] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_35.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 36/100\n",
            "148/148 [==============================] - 97s 655ms/step - loss: 1.1450 - perplexity: 2234.4099\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:43:58] <<< Saving model to trained_models/tutorial_model/epoch_36 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:43:58] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:43:58] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7604.801578 \t Average cost: 25.349339\n",
            "The sampling took: 56.921863 secs (Speed: 0.189740 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:44:55] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:44:55] Decoding beam search prediction ...\n",
            "[23/06/2020 19:44:55] Using heuristic 0\n",
            "[23/06/2020 19:44:55] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:44:55] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:44:55] Bleu_4: 9.238637447669758\n",
            "[23/06/2020 19:44:55] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:44:55] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_36.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 37/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.1368 - perplexity: 2330.8760\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:46:32] <<< Saving model to trained_models/tutorial_model/epoch_37 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:46:33] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:46:33] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7527.382926 \t Average cost: 25.091276\n",
            "The sampling took: 56.356327 secs (Speed: 0.187854 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:47:29] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:47:29] Decoding beam search prediction ...\n",
            "[23/06/2020 19:47:29] Using heuristic 0\n",
            "[23/06/2020 19:47:29] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:47:29] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:47:29] Bleu_4: 9.398888285109644\n",
            "[23/06/2020 19:47:29] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:47:29] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_37.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 38/100\n",
            "148/148 [==============================] - 96s 646ms/step - loss: 1.1283 - perplexity: 2273.6797\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:49:05] <<< Saving model to trained_models/tutorial_model/epoch_38 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:49:05] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:49:05] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7301.824656 \t Average cost: 24.339416\n",
            "The sampling took: 55.740974 secs (Speed: 0.185803 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:50:01] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:50:01] Decoding beam search prediction ...\n",
            "[23/06/2020 19:50:01] Using heuristic 0\n",
            "[23/06/2020 19:50:01] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:50:01] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:50:01] Bleu_4: 9.644573544903507\n",
            "[23/06/2020 19:50:01] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:50:01] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_38.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 39/100\n",
            "148/148 [==============================] - 96s 649ms/step - loss: 1.1196 - perplexity: 2328.6292\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:51:37] <<< Saving model to trained_models/tutorial_model/epoch_39 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:51:38] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:51:38] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7560.993551 \t Average cost: 25.203312\n",
            "The sampling took: 55.608098 secs (Speed: 0.185360 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:52:33] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:52:33] Decoding beam search prediction ...\n",
            "[23/06/2020 19:52:33] Using heuristic 0\n",
            "[23/06/2020 19:52:33] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:52:33] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:52:33] Bleu_4: 9.563931347836032\n",
            "[23/06/2020 19:52:33] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:52:34] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_39.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 40/100\n",
            "148/148 [==============================] - 100s 676ms/step - loss: 1.1164 - perplexity: 2406.5767\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:54:14] <<< Saving model to trained_models/tutorial_model/epoch_40 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:54:14] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:54:14] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7678.439138 \t Average cost: 25.594797\n",
            "The sampling took: 57.075130 secs (Speed: 0.190250 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:55:11] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:55:11] Decoding beam search prediction ...\n",
            "[23/06/2020 19:55:11] Using heuristic 0\n",
            "[23/06/2020 19:55:11] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:55:11] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:55:11] Bleu_4: 9.527379411956836\n",
            "[23/06/2020 19:55:11] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:55:11] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_40.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 41/100\n",
            "148/148 [==============================] - 95s 644ms/step - loss: 1.1088 - perplexity: 2499.4802\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:56:47] <<< Saving model to trained_models/tutorial_model/epoch_41 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:56:47] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:56:47] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7163.346158 \t Average cost: 23.877821\n",
            "The sampling took: 55.099323 secs (Speed: 0.183664 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:57:42] Prediction output 0: target_text (text)\n",
            "[23/06/2020 19:57:42] Decoding beam search prediction ...\n",
            "[23/06/2020 19:57:42] Using heuristic 0\n",
            "[23/06/2020 19:57:42] Evaluating on metric sacrebleu\n",
            "[23/06/2020 19:57:42] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 19:57:42] Bleu_4: 9.555662745521659\n",
            "[23/06/2020 19:57:42] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:57:42] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_41.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 42/100\n",
            "148/148 [==============================] - 101s 683ms/step - loss: 1.1110 - perplexity: 2576.5090\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:59:23] <<< Saving model to trained_models/tutorial_model/epoch_42 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 19:59:24] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 19:59:24] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7221.728309 \t Average cost: 24.072428\n",
            "The sampling took: 61.671511 secs (Speed: 0.205572 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:00:25] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:00:25] Decoding beam search prediction ...\n",
            "[23/06/2020 20:00:25] Using heuristic 0\n",
            "[23/06/2020 20:00:25] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:00:26] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:00:26] Bleu_4: 9.3253178810596\n",
            "[23/06/2020 20:00:26] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:00:26] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_42.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 43/100\n",
            "148/148 [==============================] - 97s 654ms/step - loss: 1.1143 - perplexity: 2595.9282\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:02:02] <<< Saving model to trained_models/tutorial_model/epoch_43 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:02:03] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:02:03] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6581.057840 \t Average cost: 21.936859\n",
            "The sampling took: 54.891424 secs (Speed: 0.182971 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:02:58] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:02:58] Decoding beam search prediction ...\n",
            "[23/06/2020 20:02:58] Using heuristic 0\n",
            "[23/06/2020 20:02:58] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:02:58] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:02:58] Bleu_4: 9.106615704026\n",
            "[23/06/2020 20:02:58] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:02:58] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_43.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 44/100\n",
            "148/148 [==============================] - 99s 667ms/step - loss: 1.1018 - perplexity: 2552.8572\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:04:37] <<< Saving model to trained_models/tutorial_model/epoch_44 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:04:37] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:04:37] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6884.035579 \t Average cost: 22.946785\n",
            "The sampling took: 58.604667 secs (Speed: 0.195349 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:05:36] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:05:36] Decoding beam search prediction ...\n",
            "[23/06/2020 20:05:36] Using heuristic 0\n",
            "[23/06/2020 20:05:36] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:05:36] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:05:36] Bleu_4: 9.437482310031912\n",
            "[23/06/2020 20:05:36] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:05:36] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_44.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 45/100\n",
            "148/148 [==============================] - 102s 689ms/step - loss: 1.0882 - perplexity: 2683.3892\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:07:18] <<< Saving model to trained_models/tutorial_model/epoch_45 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:07:18] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:07:18] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6727.515719 \t Average cost: 22.425052\n",
            "The sampling took: 56.184252 secs (Speed: 0.187281 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:08:14] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:08:14] Decoding beam search prediction ...\n",
            "[23/06/2020 20:08:14] Using heuristic 0\n",
            "[23/06/2020 20:08:14] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:08:14] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:08:14] Bleu_4: 10.061993050211811\n",
            "[23/06/2020 20:08:15] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:08:15] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_45.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 46/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.0847 - perplexity: 2779.0500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:09:52] <<< Saving model to trained_models/tutorial_model/epoch_46 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:09:53] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:09:53] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6964.952590 \t Average cost: 23.216509\n",
            "The sampling took: 56.345832 secs (Speed: 0.187819 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:10:49] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:10:49] Decoding beam search prediction ...\n",
            "[23/06/2020 20:10:49] Using heuristic 0\n",
            "[23/06/2020 20:10:49] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:10:49] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:10:49] Bleu_4: 9.739542332505714\n",
            "[23/06/2020 20:10:49] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:10:49] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_46.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 47/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.0894 - perplexity: 2663.7661\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:12:27] <<< Saving model to trained_models/tutorial_model/epoch_47 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:12:27] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:12:27] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6731.154636 \t Average cost: 22.437182\n",
            "The sampling took: 56.459270 secs (Speed: 0.188198 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:13:24] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:13:24] Decoding beam search prediction ...\n",
            "[23/06/2020 20:13:24] Using heuristic 0\n",
            "[23/06/2020 20:13:24] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:13:24] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:13:24] Bleu_4: 9.536769390261409\n",
            "[23/06/2020 20:13:24] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:13:24] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_47.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 48/100\n",
            "148/148 [==============================] - 100s 675ms/step - loss: 1.0942 - perplexity: 2686.8672\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:15:04] <<< Saving model to trained_models/tutorial_model/epoch_48 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:15:04] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:15:04] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6687.152624 \t Average cost: 22.290509\n",
            "The sampling took: 58.438456 secs (Speed: 0.194795 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:16:02] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:16:02] Decoding beam search prediction ...\n",
            "[23/06/2020 20:16:02] Using heuristic 0\n",
            "[23/06/2020 20:16:02] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:16:03] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:16:03] Bleu_4: 9.553961131561675\n",
            "[23/06/2020 20:16:03] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:16:03] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_48.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 49/100\n",
            "148/148 [==============================] - 100s 673ms/step - loss: 1.0747 - perplexity: 2617.1072\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:17:42] <<< Saving model to trained_models/tutorial_model/epoch_49 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:17:43] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:17:43] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7060.378671 \t Average cost: 23.534596\n",
            "The sampling took: 58.524377 secs (Speed: 0.195081 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:18:41] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:18:41] Decoding beam search prediction ...\n",
            "[23/06/2020 20:18:41] Using heuristic 0\n",
            "[23/06/2020 20:18:41] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:18:41] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:18:41] Bleu_4: 9.517846140216372\n",
            "[23/06/2020 20:18:41] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:18:41] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_49.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 50/100\n",
            "148/148 [==============================] - 99s 671ms/step - loss: 1.0726 - perplexity: 2812.7300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:20:21] <<< Saving model to trained_models/tutorial_model/epoch_50 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:20:21] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:20:21] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6995.438509 \t Average cost: 23.318128\n",
            "The sampling took: 57.989765 secs (Speed: 0.193299 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:21:19] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:21:19] Decoding beam search prediction ...\n",
            "[23/06/2020 20:21:19] Using heuristic 0\n",
            "[23/06/2020 20:21:19] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:21:19] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:21:19] Bleu_4: 9.733574833911993\n",
            "[23/06/2020 20:21:19] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:21:19] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_50.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 51/100\n",
            "148/148 [==============================] - 99s 666ms/step - loss: 1.0763 - perplexity: 2708.5210\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:22:58] <<< Saving model to trained_models/tutorial_model/epoch_51 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:22:58] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:22:58] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6659.630686 \t Average cost: 22.198769\n",
            "The sampling took: 56.602600 secs (Speed: 0.188675 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:23:55] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:23:55] Decoding beam search prediction ...\n",
            "[23/06/2020 20:23:55] Using heuristic 0\n",
            "[23/06/2020 20:23:55] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:23:55] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:23:55] Bleu_4: 9.862909989297473\n",
            "[23/06/2020 20:23:55] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:23:55] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_51.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 52/100\n",
            "148/148 [==============================] - 99s 670ms/step - loss: 1.0718 - perplexity: 2719.5935\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:25:34] <<< Saving model to trained_models/tutorial_model/epoch_52 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:25:34] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:25:34] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6833.267595 \t Average cost: 22.777559\n",
            "The sampling took: 57.286779 secs (Speed: 0.190956 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:26:32] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:26:32] Decoding beam search prediction ...\n",
            "[23/06/2020 20:26:32] Using heuristic 0\n",
            "[23/06/2020 20:26:32] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:26:32] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:26:32] Bleu_4: 9.543765803615528\n",
            "[23/06/2020 20:26:32] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:26:32] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_52.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 53/100\n",
            "148/148 [==============================] - 99s 667ms/step - loss: 1.0630 - perplexity: 2796.0005\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:28:11] <<< Saving model to trained_models/tutorial_model/epoch_53 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:28:11] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:28:11] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6861.457971 \t Average cost: 22.871527\n",
            "The sampling took: 56.571863 secs (Speed: 0.188573 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:29:07] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:29:07] Decoding beam search prediction ...\n",
            "[23/06/2020 20:29:07] Using heuristic 0\n",
            "[23/06/2020 20:29:07] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:29:08] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:29:08] Bleu_4: 9.317878908693817\n",
            "[23/06/2020 20:29:08] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:29:08] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_53.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 54/100\n",
            "148/148 [==============================] - 99s 667ms/step - loss: 1.0626 - perplexity: 2985.1714\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:30:46] <<< Saving model to trained_models/tutorial_model/epoch_54 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:30:47] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:30:47] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6605.033081 \t Average cost: 22.016777\n",
            "The sampling took: 56.491469 secs (Speed: 0.188305 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:31:43] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:31:43] Decoding beam search prediction ...\n",
            "[23/06/2020 20:31:43] Using heuristic 0\n",
            "[23/06/2020 20:31:43] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:31:43] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:31:43] Bleu_4: 9.575636076877691\n",
            "[23/06/2020 20:31:43] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:31:43] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_54.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 55/100\n",
            "148/148 [==============================] - 99s 668ms/step - loss: 1.0585 - perplexity: 2808.0903\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:33:22] <<< Saving model to trained_models/tutorial_model/epoch_55 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:33:23] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:33:23] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6433.370908 \t Average cost: 21.444570\n",
            "The sampling took: 55.332325 secs (Speed: 0.184441 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:34:18] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:34:18] Decoding beam search prediction ...\n",
            "[23/06/2020 20:34:18] Using heuristic 0\n",
            "[23/06/2020 20:34:18] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:34:18] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:34:18] Bleu_4: 9.411587491207714\n",
            "[23/06/2020 20:34:18] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:34:18] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_55.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 56/100\n",
            "148/148 [==============================] - 98s 659ms/step - loss: 1.0591 - perplexity: 2824.8047\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:35:56] <<< Saving model to trained_models/tutorial_model/epoch_56 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:35:56] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:35:56] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6318.478800 \t Average cost: 21.061596\n",
            "The sampling took: 55.931268 secs (Speed: 0.186438 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:36:52] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:36:52] Decoding beam search prediction ...\n",
            "[23/06/2020 20:36:52] Using heuristic 0\n",
            "[23/06/2020 20:36:52] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:36:52] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:36:52] Bleu_4: 9.443753024997639\n",
            "[23/06/2020 20:36:52] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:36:52] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_56.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 57/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.0545 - perplexity: 2706.6470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:38:30] <<< Saving model to trained_models/tutorial_model/epoch_57 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:38:31] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:38:31] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6427.394961 \t Average cost: 21.424650\n",
            "The sampling took: 54.702860 secs (Speed: 0.182343 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:39:25] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:39:25] Decoding beam search prediction ...\n",
            "[23/06/2020 20:39:25] Using heuristic 0\n",
            "[23/06/2020 20:39:25] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:39:25] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:39:25] Bleu_4: 9.491316389221636\n",
            "[23/06/2020 20:39:25] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:39:26] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_57.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 58/100\n",
            "148/148 [==============================] - 98s 664ms/step - loss: 1.0468 - perplexity: 2627.4695\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:41:04] <<< Saving model to trained_models/tutorial_model/epoch_58 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:41:04] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:41:04] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6851.614093 \t Average cost: 22.838714\n",
            "The sampling took: 57.737432 secs (Speed: 0.192458 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:42:02] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:42:02] Decoding beam search prediction ...\n",
            "[23/06/2020 20:42:02] Using heuristic 0\n",
            "[23/06/2020 20:42:02] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:42:02] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:42:02] Bleu_4: 9.474427161819968\n",
            "[23/06/2020 20:42:02] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:42:02] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_58.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 59/100\n",
            "148/148 [==============================] - 98s 664ms/step - loss: 1.0437 - perplexity: 2715.4182\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:43:40] <<< Saving model to trained_models/tutorial_model/epoch_59 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:43:41] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:43:41] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6466.864155 \t Average cost: 21.556214\n",
            "The sampling took: 55.356129 secs (Speed: 0.184520 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:44:36] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:44:36] Decoding beam search prediction ...\n",
            "[23/06/2020 20:44:36] Using heuristic 0\n",
            "[23/06/2020 20:44:36] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:44:36] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:44:36] Bleu_4: 9.606494244777767\n",
            "[23/06/2020 20:44:36] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:44:36] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_59.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 60/100\n",
            "148/148 [==============================] - 98s 665ms/step - loss: 1.0476 - perplexity: 2747.0615\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:46:15] <<< Saving model to trained_models/tutorial_model/epoch_60 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:46:15] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:46:15] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6377.123948 \t Average cost: 21.257080\n",
            "The sampling took: 55.458178 secs (Speed: 0.184861 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:47:10] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:47:10] Decoding beam search prediction ...\n",
            "[23/06/2020 20:47:10] Using heuristic 0\n",
            "[23/06/2020 20:47:10] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:47:11] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:47:11] Bleu_4: 9.603534654236219\n",
            "[23/06/2020 20:47:11] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:47:11] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_60.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 61/100\n",
            "148/148 [==============================] - 98s 659ms/step - loss: 1.0519 - perplexity: 2850.8938\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:48:48] <<< Saving model to trained_models/tutorial_model/epoch_61 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:48:49] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:48:49] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6692.961959 \t Average cost: 22.309873\n",
            "The sampling took: 57.592729 secs (Speed: 0.191976 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:49:46] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:49:46] Decoding beam search prediction ...\n",
            "[23/06/2020 20:49:46] Using heuristic 0\n",
            "[23/06/2020 20:49:46] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:49:46] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:49:46] Bleu_4: 9.611640229637231\n",
            "[23/06/2020 20:49:46] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:49:46] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_61.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 62/100\n",
            "148/148 [==============================] - 98s 660ms/step - loss: 1.0473 - perplexity: 2653.4453\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:51:24] <<< Saving model to trained_models/tutorial_model/epoch_62 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:51:24] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:51:24] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6611.452342 \t Average cost: 22.038174\n",
            "The sampling took: 57.216729 secs (Speed: 0.190722 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:52:22] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:52:22] Decoding beam search prediction ...\n",
            "[23/06/2020 20:52:22] Using heuristic 0\n",
            "[23/06/2020 20:52:22] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:52:22] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:52:22] Bleu_4: 9.552205821308075\n",
            "[23/06/2020 20:52:22] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:52:22] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_62.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 63/100\n",
            "148/148 [==============================] - 96s 651ms/step - loss: 1.0396 - perplexity: 2650.9219\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:53:58] <<< Saving model to trained_models/tutorial_model/epoch_63 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:53:59] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:53:59] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6101.206451 \t Average cost: 20.337355\n",
            "The sampling took: 54.047101 secs (Speed: 0.180157 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:54:53] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:54:53] Decoding beam search prediction ...\n",
            "[23/06/2020 20:54:53] Using heuristic 0\n",
            "[23/06/2020 20:54:53] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:54:53] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:54:53] Bleu_4: 9.779038911821125\n",
            "[23/06/2020 20:54:53] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:54:53] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_63.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 64/100\n",
            "148/148 [==============================] - 97s 656ms/step - loss: 1.0300 - perplexity: 2700.1499\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:56:30] <<< Saving model to trained_models/tutorial_model/epoch_64 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:56:30] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:56:30] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6412.066857 \t Average cost: 21.373556\n",
            "The sampling took: 55.852634 secs (Speed: 0.186175 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:57:26] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:57:26] Decoding beam search prediction ...\n",
            "[23/06/2020 20:57:26] Using heuristic 0\n",
            "[23/06/2020 20:57:26] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:57:26] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:57:26] Bleu_4: 9.758043425632913\n",
            "[23/06/2020 20:57:26] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:57:27] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_64.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 65/100\n",
            "148/148 [==============================] - 97s 652ms/step - loss: 1.0282 - perplexity: 2711.3599\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:59:03] <<< Saving model to trained_models/tutorial_model/epoch_65 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:59:03] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 20:59:03] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6194.238346 \t Average cost: 20.647461\n",
            "The sampling took: 54.726176 secs (Speed: 0.182421 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:59:58] Prediction output 0: target_text (text)\n",
            "[23/06/2020 20:59:58] Decoding beam search prediction ...\n",
            "[23/06/2020 20:59:58] Using heuristic 0\n",
            "[23/06/2020 20:59:58] Evaluating on metric sacrebleu\n",
            "[23/06/2020 20:59:58] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 20:59:58] Bleu_4: 9.183442007541506\n",
            "[23/06/2020 20:59:58] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 20:59:58] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_65.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 66/100\n",
            "148/148 [==============================] - 98s 659ms/step - loss: 1.0267 - perplexity: 2876.8372\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:01:36] <<< Saving model to trained_models/tutorial_model/epoch_66 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:01:36] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:01:36] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6051.750622 \t Average cost: 20.172502\n",
            "The sampling took: 54.599894 secs (Speed: 0.182000 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:02:31] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:02:31] Decoding beam search prediction ...\n",
            "[23/06/2020 21:02:31] Using heuristic 0\n",
            "[23/06/2020 21:02:31] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:02:31] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:02:31] Bleu_4: 8.730966632765078\n",
            "[23/06/2020 21:02:31] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:02:31] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_66.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 67/100\n",
            "148/148 [==============================] - 97s 655ms/step - loss: 1.0275 - perplexity: 2898.7351\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:04:08] <<< Saving model to trained_models/tutorial_model/epoch_67 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:04:08] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:04:08] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6277.127036 \t Average cost: 20.923757\n",
            "The sampling took: 56.731758 secs (Speed: 0.189106 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:05:05] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:05:05] Decoding beam search prediction ...\n",
            "[23/06/2020 21:05:05] Using heuristic 0\n",
            "[23/06/2020 21:05:05] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:05:05] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:05:05] Bleu_4: 9.23943429962001\n",
            "[23/06/2020 21:05:05] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:05:05] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_67.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 68/100\n",
            "148/148 [==============================] - 98s 661ms/step - loss: 1.0289 - perplexity: 2843.2102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:06:43] <<< Saving model to trained_models/tutorial_model/epoch_68 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:06:44] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:06:44] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6162.271337 \t Average cost: 20.540904\n",
            "The sampling took: 56.240916 secs (Speed: 0.187470 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:07:40] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:07:40] Decoding beam search prediction ...\n",
            "[23/06/2020 21:07:40] Using heuristic 0\n",
            "[23/06/2020 21:07:40] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:07:40] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:07:40] Bleu_4: 9.37639552652251\n",
            "[23/06/2020 21:07:40] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:07:40] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_68.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 69/100\n",
            "148/148 [==============================] - 97s 656ms/step - loss: 1.0251 - perplexity: 2883.1736\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:09:17] <<< Saving model to trained_models/tutorial_model/epoch_69 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:09:17] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:09:17] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6160.427656 \t Average cost: 20.534759\n",
            "The sampling took: 53.483588 secs (Speed: 0.178279 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:10:11] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:10:11] Decoding beam search prediction ...\n",
            "[23/06/2020 21:10:11] Using heuristic 0\n",
            "[23/06/2020 21:10:11] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:10:11] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:10:11] Bleu_4: 8.84733102175906\n",
            "[23/06/2020 21:10:11] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:10:11] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_69.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 70/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.0179 - perplexity: 2605.8940\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:11:48] <<< Saving model to trained_models/tutorial_model/epoch_70 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:11:48] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:11:48] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6306.627819 \t Average cost: 21.022093\n",
            "The sampling took: 53.574539 secs (Speed: 0.178582 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:12:42] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:12:42] Decoding beam search prediction ...\n",
            "[23/06/2020 21:12:42] Using heuristic 0\n",
            "[23/06/2020 21:12:42] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:12:42] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:12:42] Bleu_4: 8.81828893671904\n",
            "[23/06/2020 21:12:42] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:12:42] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_70.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 71/100\n",
            "148/148 [==============================] - 96s 652ms/step - loss: 1.0088 - perplexity: 2724.7000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:14:18] <<< Saving model to trained_models/tutorial_model/epoch_71 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:14:19] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:14:19] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5998.365873 \t Average cost: 19.994553\n",
            "The sampling took: 53.223711 secs (Speed: 0.177412 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:15:12] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:15:12] Decoding beam search prediction ...\n",
            "[23/06/2020 21:15:12] Using heuristic 0\n",
            "[23/06/2020 21:15:12] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:15:12] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:15:12] Bleu_4: 8.604671210607524\n",
            "[23/06/2020 21:15:12] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:15:12] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_71.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 72/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.0025 - perplexity: 2767.4663\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:16:49] <<< Saving model to trained_models/tutorial_model/epoch_72 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:16:49] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:16:49] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6208.964130 \t Average cost: 20.696547\n",
            "The sampling took: 54.398394 secs (Speed: 0.181328 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:17:44] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:17:44] Decoding beam search prediction ...\n",
            "[23/06/2020 21:17:44] Using heuristic 0\n",
            "[23/06/2020 21:17:44] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:17:44] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:17:44] Bleu_4: 8.66879387600759\n",
            "[23/06/2020 21:17:44] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:17:44] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_72.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 73/100\n",
            "148/148 [==============================] - 97s 656ms/step - loss: 1.0022 - perplexity: 2718.7256\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:19:21] <<< Saving model to trained_models/tutorial_model/epoch_73 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:19:21] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:19:21] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6467.968415 \t Average cost: 21.559895\n",
            "The sampling took: 54.626530 secs (Speed: 0.182088 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:20:16] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:20:16] Decoding beam search prediction ...\n",
            "[23/06/2020 21:20:16] Using heuristic 0\n",
            "[23/06/2020 21:20:16] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:20:16] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:20:16] Bleu_4: 9.39695142995592\n",
            "[23/06/2020 21:20:16] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:20:16] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_73.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 74/100\n",
            "148/148 [==============================] - 96s 650ms/step - loss: 1.0078 - perplexity: 2894.9912\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:21:52] <<< Saving model to trained_models/tutorial_model/epoch_74 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:21:53] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:21:53] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6469.511178 \t Average cost: 21.565037\n",
            "The sampling took: 55.329843 secs (Speed: 0.184433 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:22:48] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:22:48] Decoding beam search prediction ...\n",
            "[23/06/2020 21:22:48] Using heuristic 0\n",
            "[23/06/2020 21:22:48] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:22:48] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:22:48] Bleu_4: 9.447016121206167\n",
            "[23/06/2020 21:22:48] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:22:48] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_74.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 75/100\n",
            "148/148 [==============================] - 97s 653ms/step - loss: 1.0102 - perplexity: 2738.7671\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:24:25] <<< Saving model to trained_models/tutorial_model/epoch_75 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:24:25] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:24:25] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5896.995193 \t Average cost: 19.656651\n",
            "The sampling took: 52.402148 secs (Speed: 0.174674 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:25:18] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:25:18] Decoding beam search prediction ...\n",
            "[23/06/2020 21:25:18] Using heuristic 0\n",
            "[23/06/2020 21:25:18] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:25:18] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:25:18] Bleu_4: 9.099257957004719\n",
            "[23/06/2020 21:25:18] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:25:18] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_75.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 76/100\n",
            "148/148 [==============================] - 97s 655ms/step - loss: 1.0111 - perplexity: 2981.3669\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:26:55] <<< Saving model to trained_models/tutorial_model/epoch_76 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:26:55] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:26:55] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6319.779707 \t Average cost: 21.065932\n",
            "The sampling took: 54.582633 secs (Speed: 0.181942 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:27:50] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:27:50] Decoding beam search prediction ...\n",
            "[23/06/2020 21:27:50] Using heuristic 0\n",
            "[23/06/2020 21:27:50] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:27:50] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:27:50] Bleu_4: 9.212396217888422\n",
            "[23/06/2020 21:27:50] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:27:50] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_76.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 77/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 1.0073 - perplexity: 3097.5583\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:29:27] <<< Saving model to trained_models/tutorial_model/epoch_77 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:29:27] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:29:27] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6293.377257 \t Average cost: 20.977924\n",
            "The sampling took: 56.155050 secs (Speed: 0.187184 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:30:24] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:30:24] Decoding beam search prediction ...\n",
            "[23/06/2020 21:30:24] Using heuristic 0\n",
            "[23/06/2020 21:30:24] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:30:24] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:30:24] Bleu_4: 9.343872881552064\n",
            "[23/06/2020 21:30:24] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:30:24] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_77.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 78/100\n",
            "148/148 [==============================] - 97s 655ms/step - loss: 1.0056 - perplexity: 2987.8167\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:32:01] <<< Saving model to trained_models/tutorial_model/epoch_78 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:32:01] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:32:01] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6283.186274 \t Average cost: 20.943954\n",
            "The sampling took: 56.533762 secs (Speed: 0.188446 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:32:58] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:32:58] Decoding beam search prediction ...\n",
            "[23/06/2020 21:32:58] Using heuristic 0\n",
            "[23/06/2020 21:32:58] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:32:58] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:32:58] Bleu_4: 8.794582426756957\n",
            "[23/06/2020 21:32:58] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:32:58] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_78.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 79/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.0080 - perplexity: 3043.4187\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:34:36] <<< Saving model to trained_models/tutorial_model/epoch_79 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:34:36] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:34:36] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6101.494420 \t Average cost: 20.338315\n",
            "The sampling took: 55.606883 secs (Speed: 0.185356 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:35:32] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:35:32] Decoding beam search prediction ...\n",
            "[23/06/2020 21:35:32] Using heuristic 0\n",
            "[23/06/2020 21:35:32] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:35:32] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:35:32] Bleu_4: 9.053040537018392\n",
            "[23/06/2020 21:35:32] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:35:32] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_79.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 80/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 1.0046 - perplexity: 2823.8945\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:37:10] <<< Saving model to trained_models/tutorial_model/epoch_80 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:37:10] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:37:10] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6144.719332 \t Average cost: 20.482398\n",
            "The sampling took: 55.500585 secs (Speed: 0.185002 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:38:06] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:38:06] Decoding beam search prediction ...\n",
            "[23/06/2020 21:38:06] Using heuristic 0\n",
            "[23/06/2020 21:38:06] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:38:06] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:38:06] Bleu_4: 9.189694536238882\n",
            "[23/06/2020 21:38:06] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:38:06] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_80.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 81/100\n",
            "148/148 [==============================] - 101s 683ms/step - loss: 0.9959 - perplexity: 2991.3391\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:39:47] <<< Saving model to trained_models/tutorial_model/epoch_81 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:39:47] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:39:47] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6086.847000 \t Average cost: 20.289490\n",
            "The sampling took: 57.279500 secs (Speed: 0.190932 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:40:45] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:40:45] Decoding beam search prediction ...\n",
            "[23/06/2020 21:40:45] Using heuristic 0\n",
            "[23/06/2020 21:40:45] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:40:45] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:40:45] Bleu_4: 9.07218452750472\n",
            "[23/06/2020 21:40:45] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:40:45] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_81.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 82/100\n",
            "148/148 [==============================] - 102s 686ms/step - loss: 0.9943 - perplexity: 2903.3022\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:42:26] <<< Saving model to trained_models/tutorial_model/epoch_82 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:42:27] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:42:27] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6177.125583 \t Average cost: 20.590419\n",
            "The sampling took: 58.968189 secs (Speed: 0.196561 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:43:26] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:43:26] Decoding beam search prediction ...\n",
            "[23/06/2020 21:43:26] Using heuristic 0\n",
            "[23/06/2020 21:43:26] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:43:26] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:43:26] Bleu_4: 9.170313282570525\n",
            "[23/06/2020 21:43:26] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:43:26] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_82.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 83/100\n",
            "148/148 [==============================] - 106s 713ms/step - loss: 0.9837 - perplexity: 3108.0173\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:45:12] <<< Saving model to trained_models/tutorial_model/epoch_83 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:45:12] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:45:12] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6098.939877 \t Average cost: 20.329800\n",
            "The sampling took: 60.912996 secs (Speed: 0.203043 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:46:13] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:46:13] Decoding beam search prediction ...\n",
            "[23/06/2020 21:46:13] Using heuristic 0\n",
            "[23/06/2020 21:46:13] Evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:46:13] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:46:13] Bleu_4: 8.732421019345175\n",
            "[23/06/2020 21:46:13] Done evaluating on metric sacrebleu\n",
            "[23/06/2020 21:46:13] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_83.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 84/100\n",
            "148/148 [==============================] - 108s 727ms/step - loss: 0.9869 - perplexity: 2808.1289\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:48:01] <<< Saving model to trained_models/tutorial_model/epoch_84 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:48:01] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:48:01] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6045.667115 \t Average cost: 20.152224\n",
            "The sampling took: 63.255590 secs (Speed: 0.210852 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:49:04] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:49:04] Decoding beam search prediction ...\n",
            "[23/06/2020 21:49:04] Using heuristic 0\n",
            "[23/06/2020 21:49:04] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:49:04] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:49:04] Bleu_4: 8.974014349501672\n",
            "[23/06/2020 21:49:04] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:49:05] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_84.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 85/100\n",
            "148/148 [==============================] - 110s 744ms/step - loss: 0.9901 - perplexity: 2975.9551\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:50:55] <<< Saving model to trained_models/tutorial_model/epoch_85 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:50:55] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:50:55] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5873.839294 \t Average cost: 19.579464\n",
            "The sampling took: 61.190765 secs (Speed: 0.203969 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:51:56] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:51:56] Decoding beam search prediction ...\n",
            "[23/06/2020 21:51:56] Using heuristic 0\n",
            "[23/06/2020 21:51:56] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:51:56] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:51:56] Bleu_4: 9.002472832146225\n",
            "[23/06/2020 21:51:56] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:51:56] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_85.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 86/100\n",
            "148/148 [==============================] - 105s 708ms/step - loss: 0.9938 - perplexity: 2935.5808\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:53:41] <<< Saving model to trained_models/tutorial_model/epoch_86 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:53:42] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:53:42] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5916.399945 \t Average cost: 19.721333\n",
            "The sampling took: 57.881952 secs (Speed: 0.192940 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:54:40] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:54:40] Decoding beam search prediction ...\n",
            "[23/06/2020 21:54:40] Using heuristic 0\n",
            "[23/06/2020 21:54:40] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:54:40] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:54:40] Bleu_4: 8.848745492954752\n",
            "[23/06/2020 21:54:40] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:54:40] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_86.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 87/100\n",
            "148/148 [==============================] - 101s 684ms/step - loss: 0.9878 - perplexity: 2980.5249\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:56:21] <<< Saving model to trained_models/tutorial_model/epoch_87 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:56:21] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:56:21] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5748.328460 \t Average cost: 19.161095\n",
            "The sampling took: 55.054741 secs (Speed: 0.183516 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:57:16] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:57:16] Decoding beam search prediction ...\n",
            "[23/06/2020 21:57:16] Using heuristic 0\n",
            "[23/06/2020 21:57:16] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:57:17] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:57:17] Bleu_4: 8.525386782383531\n",
            "[23/06/2020 21:57:17] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:57:17] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_87.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 88/100\n",
            "148/148 [==============================] - 100s 674ms/step - loss: 0.9831 - perplexity: 2864.7427\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:58:56] <<< Saving model to trained_models/tutorial_model/epoch_88 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:58:57] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 21:58:57] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5922.257424 \t Average cost: 19.740858\n",
            "The sampling took: 57.405919 secs (Speed: 0.191353 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:59:54] Prediction output 0: target_text (text)\n",
            "[23/06/2020 21:59:54] Decoding beam search prediction ...\n",
            "[23/06/2020 21:59:54] Using heuristic 0\n",
            "[23/06/2020 21:59:54] Evaluating on metric sacrebleu\n",
            "[23/06/2020 21:59:54] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 21:59:54] Bleu_4: 9.58822639286586\n",
            "[23/06/2020 21:59:54] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 21:59:54] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_88.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 89/100\n",
            "148/148 [==============================] - 101s 684ms/step - loss: 0.9859 - perplexity: 2934.1719\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:01:36] <<< Saving model to trained_models/tutorial_model/epoch_89 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:01:36] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:01:36] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6165.121671 \t Average cost: 20.550406\n",
            "The sampling took: 57.097190 secs (Speed: 0.190324 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:02:33] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:02:33] Decoding beam search prediction ...\n",
            "[23/06/2020 22:02:33] Using heuristic 0\n",
            "[23/06/2020 22:02:33] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:02:33] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:02:33] Bleu_4: 9.040111659986389\n",
            "[23/06/2020 22:02:33] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:02:33] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_89.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 90/100\n",
            "148/148 [==============================] - 99s 667ms/step - loss: 0.9889 - perplexity: 2997.4148\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:04:12] <<< Saving model to trained_models/tutorial_model/epoch_90 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:04:12] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:04:12] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5880.724283 \t Average cost: 19.602414\n",
            "The sampling took: 55.396782 secs (Speed: 0.184656 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:05:08] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:05:08] Decoding beam search prediction ...\n",
            "[23/06/2020 22:05:08] Using heuristic 0\n",
            "[23/06/2020 22:05:08] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:05:08] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:05:08] Bleu_4: 8.731660827076784\n",
            "[23/06/2020 22:05:08] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:05:08] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_90.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 91/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 0.9750 - perplexity: 2912.6252\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:06:46] <<< Saving model to trained_models/tutorial_model/epoch_91 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:06:46] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:06:46] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5822.410431 \t Average cost: 19.408035\n",
            "The sampling took: 53.709510 secs (Speed: 0.179032 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:07:40] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:07:40] Decoding beam search prediction ...\n",
            "[23/06/2020 22:07:40] Using heuristic 0\n",
            "[23/06/2020 22:07:40] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:07:40] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:07:40] Bleu_4: 9.182386914514941\n",
            "[23/06/2020 22:07:40] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:07:40] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_91.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 92/100\n",
            "148/148 [==============================] - 98s 662ms/step - loss: 0.9681 - perplexity: 2929.5437\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:09:18] <<< Saving model to trained_models/tutorial_model/epoch_92 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:09:19] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:09:19] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 6144.773901 \t Average cost: 20.482580\n",
            "The sampling took: 57.100949 secs (Speed: 0.190336 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:10:16] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:10:16] Decoding beam search prediction ...\n",
            "[23/06/2020 22:10:16] Using heuristic 0\n",
            "[23/06/2020 22:10:16] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:10:16] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:10:16] Bleu_4: 8.74976563647757\n",
            "[23/06/2020 22:10:16] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:10:16] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_92.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 93/100\n",
            "148/148 [==============================] - 98s 664ms/step - loss: 0.9690 - perplexity: 2954.0608\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:11:54] <<< Saving model to trained_models/tutorial_model/epoch_93 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:11:55] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:11:55] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5917.168413 \t Average cost: 19.723895\n",
            "The sampling took: 53.695745 secs (Speed: 0.178986 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:12:48] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:12:48] Decoding beam search prediction ...\n",
            "[23/06/2020 22:12:48] Using heuristic 0\n",
            "[23/06/2020 22:12:48] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:12:48] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:12:48] Bleu_4: 8.684695794240673\n",
            "[23/06/2020 22:12:48] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:12:49] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_93.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 94/100\n",
            "148/148 [==============================] - 97s 657ms/step - loss: 0.9691 - perplexity: 3028.1675\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:14:26] <<< Saving model to trained_models/tutorial_model/epoch_94 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:14:26] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:14:26] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5788.777311 \t Average cost: 19.295924\n",
            "The sampling took: 53.877102 secs (Speed: 0.179590 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:15:20] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:15:20] Decoding beam search prediction ...\n",
            "[23/06/2020 22:15:20] Using heuristic 0\n",
            "[23/06/2020 22:15:20] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:15:20] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:15:20] Bleu_4: 8.791515260035075\n",
            "[23/06/2020 22:15:20] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:15:20] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_94.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 95/100\n",
            "148/148 [==============================] - 98s 661ms/step - loss: 0.9695 - perplexity: 3088.2615\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:16:58] <<< Saving model to trained_models/tutorial_model/epoch_95 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:16:58] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:16:58] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5730.832749 \t Average cost: 19.102776\n",
            "The sampling took: 53.488774 secs (Speed: 0.178296 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:17:52] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:17:52] Decoding beam search prediction ...\n",
            "[23/06/2020 22:17:52] Using heuristic 0\n",
            "[23/06/2020 22:17:52] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:17:52] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:17:52] Bleu_4: 8.96410765988919\n",
            "[23/06/2020 22:17:52] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:17:52] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_95.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 96/100\n",
            "148/148 [==============================] - 97s 654ms/step - loss: 0.9766 - perplexity: 3177.4482\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:19:29] <<< Saving model to trained_models/tutorial_model/epoch_96 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:19:29] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:19:29] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5859.004258 \t Average cost: 19.530014\n",
            "The sampling took: 53.970437 secs (Speed: 0.179901 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:20:23] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:20:23] Decoding beam search prediction ...\n",
            "[23/06/2020 22:20:23] Using heuristic 0\n",
            "[23/06/2020 22:20:23] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:20:23] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:20:23] Bleu_4: 8.497949184258939\n",
            "[23/06/2020 22:20:23] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:20:23] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_96.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 97/100\n",
            "148/148 [==============================] - 97s 659ms/step - loss: 0.9783 - perplexity: 3091.3984\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:22:01] <<< Saving model to trained_models/tutorial_model/epoch_97 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:22:01] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:22:01] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5979.141929 \t Average cost: 19.930473\n",
            "The sampling took: 53.938205 secs (Speed: 0.179794 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:22:55] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:22:55] Decoding beam search prediction ...\n",
            "[23/06/2020 22:22:55] Using heuristic 0\n",
            "[23/06/2020 22:22:55] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:22:55] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:22:55] Bleu_4: 8.76652748818501\n",
            "[23/06/2020 22:22:55] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:22:55] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_97.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 98/100\n",
            "148/148 [==============================] - 98s 664ms/step - loss: 0.9681 - perplexity: 3055.3955\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:24:34] <<< Saving model to trained_models/tutorial_model/epoch_98 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:24:34] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:24:34] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5897.767582 \t Average cost: 19.659225\n",
            "The sampling took: 54.809759 secs (Speed: 0.182699 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:25:29] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:25:29] Decoding beam search prediction ...\n",
            "[23/06/2020 22:25:29] Using heuristic 0\n",
            "[23/06/2020 22:25:29] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:25:29] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:25:29] Bleu_4: 8.824969231903355\n",
            "[23/06/2020 22:25:29] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:25:29] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_98.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 99/100\n",
            "148/148 [==============================] - 97s 658ms/step - loss: 0.9622 - perplexity: 3003.1848\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:27:07] <<< Saving model to trained_models/tutorial_model/epoch_99 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:27:07] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:27:07] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5784.373677 \t Average cost: 19.281246\n",
            "The sampling took: 53.229352 secs (Speed: 0.177431 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:28:00] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:28:00] Decoding beam search prediction ...\n",
            "[23/06/2020 22:28:00] Using heuristic 0\n",
            "[23/06/2020 22:28:00] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:28:00] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:28:00] Bleu_4: 9.070217282033854\n",
            "[23/06/2020 22:28:00] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:28:00] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_99.jpg >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 100/100\n",
            "148/148 [==============================] - 96s 646ms/step - loss: 0.9611 - perplexity: 3020.3257\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:29:36] <<< Saving model to trained_models/tutorial_model/epoch_100 ... >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:29:36] <<< Model saved >>>\n",
            "\n",
            "[23/06/2020 22:29:36] <<< Predicting outputs of val set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 5478.476755 \t Average cost: 18.261589\n",
            "The sampling took: 52.739933 secs (Speed: 0.175800 sec/sample)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:30:29] Prediction output 0: target_text (text)\n",
            "[23/06/2020 22:30:29] Decoding beam search prediction ...\n",
            "[23/06/2020 22:30:29] Using heuristic 0\n",
            "[23/06/2020 22:30:29] Evaluating on metric sacrebleu\n",
            "[23/06/2020 22:30:29] Computing SacreBleu scores on the val split...\n",
            "[23/06/2020 22:30:29] Bleu_4: 7.996934813441331\n",
            "[23/06/2020 22:30:29] Done evaluating on metric sacrebleu\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[23/06/2020 22:30:29] \n",
            "<<< Progress plot saved in trained_models/tutorial_model/epoch_100.jpg >>>\n",
            "[23/06/2020 22:30:29] <<< Finished training model >>>\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R_juptEcN403",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "8fe6f7bf-b3b9-4e26-c803-e915adc5b699"
      },
      "source": [
        "dataset.setInput('examples/task/test.ur',\n",
        "            'test',\n",
        "            type='text',\n",
        "            id='source_text',\n",
        "            pad_on_batch=True,\n",
        "            tokenization='tokenize_none',\n",
        "            fill='end',\n",
        "            max_text_len=50,\n",
        "            min_occ=0)\n",
        "\n",
        "dataset.setInput(None,\n",
        "            'test',\n",
        "            type='ghost',\n",
        "            id='state_below',\n",
        "            required=False)\n",
        "\n",
        "dataset.setRawInput('examples/task/test.ur',\n",
        "              'test',\n",
        "              type='file-name',\n",
        "              id='raw_source_text',\n",
        "              overwrite_split=True)\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:44:59] \tApplying tokenization function: \"tokenize_none\".\n",
            "[24/06/2020 05:44:59] Loaded \"test\" set inputs of data_type \"text\" with data_id \"source_text\" and length 257.\n",
            "[24/06/2020 05:44:59] Loaded \"test\" set inputs of data_type \"ghost\" with data_id \"state_below\" and length 257.\n",
            "[24/06/2020 05:44:59] Loaded \"test\" set inputs of type \"file-name\" with id \"raw_source_text\".\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U5HLsqHizw6k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "23c00484-f0cd-4ff3-ad9d-5bfad559b9eb"
      },
      "source": [
        "params['INPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[params['INPUTS_IDS_DATASET'][0]]\n",
        "params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[params['OUTPUTS_IDS_DATASET'][0]]\n",
        "\n",
        "# Load model\n",
        "nmt_model = loadModel('trained_models/tutorial_model', 33)\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:45:03] <<< Loading model from trained_models/tutorial_model/epoch_33_Model_Wrapper.pkl ... >>>\n",
            "[24/06/2020 05:45:03] <<< Loading model from trained_models/tutorial_model/epoch_33.h5 ... >>>\n",
            "[24/06/2020 05:45:06] <<< Loading optimized model... >>>\n",
            "[24/06/2020 05:45:21] <<< Optimized model loaded. >>>\n",
            "[24/06/2020 05:45:21] <<< Model loaded in 17.988 seconds. >>>\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfCEDFQs1fe3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "63db2613-f114-4a3f-9902-0d182b3b4071"
      },
      "source": [
        "is_transformer = params.get('ATTEND_ON_OUTPUT', 'transformer' in params['MODEL_TYPE'].lower())\n",
        "\n",
        "params_prediction = {\n",
        "    'language': 'en',\n",
        "    'tokenize_f': eval('dataset.' + 'tokenize_none'),\n",
        "    'beam_size': 12,\n",
        "    'optimized_search': True,\n",
        "    'model_inputs': params['INPUTS_IDS_MODEL'],\n",
        "    'model_outputs': params['OUTPUTS_IDS_MODEL'],\n",
        "    'dataset_inputs':  params['INPUTS_IDS_DATASET'],\n",
        "    'dataset_outputs':  params['OUTPUTS_IDS_DATASET'],\n",
        "    'n_parallel_loaders': 1,\n",
        "    'maxlen': 50,\n",
        "    'normalize_probs': True,\n",
        "    'pos_unk': True and not is_transformer,\n",
        "    'heuristic': 0,\n",
        "    'state_below_maxlen': -1,\n",
        "    'predict_on_sets': ['test'],\n",
        "    'verbose': 0,\n",
        "    'attend_on_output': is_transformer\n",
        "  }\n",
        "predictions = nmt_model.predictBeamSearchNet(dataset, params_prediction)['test']"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "[24/06/2020 05:48:51] <<< Predicting outputs of test set >>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Total cost: 7308.658398 \t Average cost: 28.438360\n",
            "The sampling took: 140.223336 secs (Speed: 0.545616 sec/sample)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0ww-sEz1ob8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1a035e00-8771-4960-b714-64e78784b1a0"
      },
      "source": [
        "from keras_wrapper.utils import decode_predictions_beam_search\n",
        "vocab = dataset.vocabulary['target_text']['idx2words']\n",
        "samples = predictions['samples'] # Get word indices from the samples.\n",
        "\n",
        "predictions = decode_predictions_beam_search(samples,  \n",
        "                                             vocab,\n",
        "                                             verbose=params['VERBOSE'])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:51:22] Decoding beam search prediction ...\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lTB3GcoV16tu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "5775440b-2202-4f0d-f95d-5f1bf6bb04f3"
      },
      "source": [
        "filepath = 'test.pred'\n",
        "from keras_wrapper.extra.read_write import list2file\n",
        "list2file(filepath, predictions)\n",
        "!head -n 4 test.pred"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Neither entered into the way into the people , but took up a great way into the way to the poor ; and not as he had been a season , as they ought to ask that he written , saying , It is a season , lest they\n",
            "There shall be the days of those days ; but they shall not see : and they enter into their own : and they persecute us into death ; and he walked against them .\n",
            "And cast them out of the fame of them , and they cast hold for their garments , and they have taken up a crown of their souls . And when they were an hungred , and cut about their own words .\n",
            "That there were afar off , and an hair , and ten , and judgeth according to the same , and took the same things .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1DEPvX542Omi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "83e690c1-eeba-432e-cd8a-8ad221da885f"
      },
      "source": [
        "dataset.setOutput('examples/task/test.en',\n",
        "             'test',\n",
        "             type='text',\n",
        "             id='target_text',\n",
        "             pad_on_batch=True,\n",
        "             tokenization='tokenize_none',\n",
        "             sample_weights=True,\n",
        "             max_text_len=50,\n",
        "             max_words=0)\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:51:29] \tApplying tokenization function: \"tokenize_none\".\n",
            "[24/06/2020 05:51:29] Loaded \"test\" set outputs of data_type \"text\" with data_id \"target_text\" and length 257.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3EFv45T2jYY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "6d228a19-0ddb-43b0-ffee-d2854938a523"
      },
      "source": [
        "keep_n_captions(dataset, repeat=1, n=1, set_names=['test'])"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:51:32] Keeping 1 captions per input on the test set.\n",
            "[24/06/2020 05:51:32] Samples reduced to 257 in test set.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zAumv2Pp2YsE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "fc4392df-dd24-491b-c503-08bdc3729e66"
      },
      "source": [
        "\n",
        "from keras_wrapper.extra.evaluation import select\n",
        "metric = 'sacrebleu'\n",
        "# Apply sampling\n",
        "extra_vars = dict()\n",
        "extra_vars['tokenize_f'] = eval('dataset.' + 'tokenize_none')\n",
        "extra_vars['language'] = params['TRG_LAN']\n",
        "extra_vars['test'] = dict()\n",
        "extra_vars['test']['references'] = dataset.extra_variables['test']['target_text']\n",
        "metrics = select[metric](pred_list=predictions,\n",
        "                                          verbose=1,\n",
        "                                          extra_vars=extra_vars,\n",
        "                                          split='test')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[24/06/2020 05:51:35] Computing SacreBleu scores on the test split...\n",
            "[24/06/2020 05:51:35] Bleu_4: 7.300801982565925\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pXPtJail2n3Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}