many_to_one_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "many_to_one_demo.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TOYI-7PAzcJm",
        "colab_type": "text"
      },
      "source": [
        "## Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ESsGAvQyVRI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "166d809c-bafc-402e-e7ec-7bae81931c6b"
      },
      "source": [
        "!pip uninstall -y tensorflow numpy && pip install tensorflow-gpu==1.14.0 tf_sentencepiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling tensorflow-1.14.0:\n",
            "  Successfully uninstalled tensorflow-1.14.0\n",
            "Uninstalling numpy-1.16.5:\n",
            "  Successfully uninstalled numpy-1.16.5\n",
            "Collecting tensorflow-gpu==1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/04/43153bfdfcf6c9a4c38ecdb971ca9a75b9a791bb69a764d652c359aca504/tensorflow_gpu-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (377.0MB)\n",
            "\u001b[K     |████████████████████████████████| 377.0MB 87kB/s \n",
            "\u001b[?25hCollecting tf_sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dc/2c/20800032089a9271757921f3adc1f2c7ec2d294ec9fa07b3115fab9d27c2/tf_sentencepiece-0.1.83-py2.py3-none-manylinux1_x86_64.whl (2.7MB)\n",
            "\u001b[K     |████████████████████████████████| 2.7MB 31.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.15.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.0.8)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (3.7.1)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.33.6)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.11.2)\n",
            "Collecting numpy<2.0,>=1.14.5 (from tensorflow-gpu==1.14.0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl (20.4MB)\n",
            "\u001b[K     |████████████████████████████████| 20.4MB 34.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.2.2)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.1.7)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.12.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (0.15.6)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (41.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (3.1.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.14.0) (2.8.0)\n",
            "\u001b[31mERROR: stable-baselines 2.2.1 requires tensorflow>=1.5.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: magenta 0.3.19 requires tensorflow>=1.12.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: fancyimpute 0.4.3 requires tensorflow, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numpy, tensorflow-gpu, tf-sentencepiece\n",
            "Successfully installed numpy-1.17.2 tensorflow-gpu-1.14.0 tf-sentencepiece-0.1.83\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8GtU8Uupt4d3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "9c633d19-4ca9-4739-f6e6-3eda7633f3c2"
      },
      "source": [
        "!curl -o many_to_one.zip https://dl.dropboxusercontent.com/s/xmroc5u6k17t5ds/many_to_one.zip?dl=0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  113M  100  113M    0     0  33.4M      0  0:00:03  0:00:03 --:--:-- 33.3M\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oXvTRqdruAcs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q many_to_one.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QK20qqzwuMbe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "a0a823dd-c207-47be-ad50-175875d4bc4f"
      },
      "source": [
        "!curl -o mlt.zip https://codeload.github.com/suyash/mlt/zip/master"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100 24327    0 24327    0     0  60066      0 --:--:-- --:--:-- --:--:-- 60066\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vijlcNtKvOEf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q mlt.zip && mv mlt-master/mlt ./"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fBfrdS7Zzrsw",
        "colab_type": "text"
      },
      "source": [
        "## Many to One Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uj-XG8o2wRn4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tf_sentencepiece as tfs\n",
        "\n",
        "from mlt.evaluation import predict\n",
        "from mlt.layers import Attention, ConditionalNormalization, Gelu, MultiplyConstant, PaddingAndLookaheadMask, PaddingMask, PositionalEncoding"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OaWArujqwYAN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d1738b7d-e4c9-431f-d912-c5853a727438"
      },
      "source": [
        "tf.__version__"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.14.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qsp6FaIMwaXM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess = tf.InteractiveSession()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "31v1oiwCwp36",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "en_model_file = \"final/sentencepiece/para_crawl/ende_plain_text/models/unigram/8192/a.model\"\n",
        "de_model_file = \"final/sentencepiece/para_crawl/ende_plain_text/models/unigram/8192/b.model\"\n",
        "fr_model_file = \"final/sentencepiece/para_crawl/enfr_plain_text/models/unigram/8192/b.model\"\n",
        "es_model_file = \"final/sentencepiece/para_crawl/enes_plain_text/models/unigram/8192/b.model\"\n",
        "it_model_file = \"final/sentencepiece/para_crawl/enit_plain_text/models/unigram/8192/b.model\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjzaZ3WHwuvg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "en_offset = 0\n",
        "fr_offset = 8192\n",
        "de_offset = fr_offset + 8192\n",
        "es_offset = de_offset + 8192\n",
        "it_offset = es_offset + 8192"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8-Ogtw7wwmg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "outputId": "1c61208b-7128-4adf-b3d9-4e3fe0b9ddb4"
      },
      "source": [
        "with sess.as_default():\n",
        "    model = tf.keras.experimental.load_from_saved_model(\"final/model114\", custom_objects={\n",
        "        \"MultiplyConstant\": MultiplyConstant,\n",
        "        \"PositionalEncoding\": PositionalEncoding,\n",
        "        \"PaddingMask\": PaddingMask,\n",
        "        \"PaddingAndLookaheadMask\": PaddingAndLookaheadMask,\n",
        "        \"Attention\": Attention,\n",
        "        \"ConditionalNormalization\": ConditionalNormalization,\n",
        "        \"Gelu\": Gelu,\n",
        "    })"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Imi04DzkzwH-",
        "colab_type": "text"
      },
      "source": [
        "### German to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PNEENWUpw0eW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Hier finden Sie die Antworten, die Sie brauchen.\"], model_file=de_model_file, add_bos=True, add_eos=True)[0] + de_offset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jkkhBxa0w6SA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "outputId": "af02afc9-fe0a-4956-fa72-4a72310a0ce2"
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0]),\n",
        "    bos_id=en_offset + 1,\n",
        "    eos_id=en_offset + 2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensor2tensor/utils/beam_search.py:727: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:2403: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tPZG8cM8x49W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lTNMZ_t4yEXl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbSGzHFXzWN8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "00bda53a-92cc-44a2-c0c9-54cd56a9d525"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=en_model_file))], probs_[0]"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Here you find the answers you need.',\n",
              "  'Here you find the answers you need here.',\n",
              "  'Click here to find the answers you need.',\n",
              "  'Here you will find the answers you need.',\n",
              "  'Here you can find the answers you need.'],\n",
              " array([0.700188  , 0.41780248, 0.37256515, 0.29034427, 0.26026225],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OOq4noyGz9Gh",
        "colab_type": "text"
      },
      "source": [
        "### French to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9NGdA_zxzYkV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Vous pouvez trouver les réponses dont vous avez besoin ici.\"], model_file=fr_model_file, add_bos=True, add_eos=True)[0] + fr_offset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E1tsQmG30G7F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0]),\n",
        "    bos_id=en_offset + 1,\n",
        "    eos_id=en_offset + 2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uMZQ8XcK0IaG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mb2B2o_F0LiK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "je4U3_uD0NgB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "b56bd056-aae3-43f0-e79a-5be3cc3b3ed4"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=en_model_file))], probs_[0]"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['You can find the answers you need.',\n",
              "  'You can find the answers you need here.',\n",
              "  'You can find the answers that you need.',\n",
              "  'You may find the answers you need.',\n",
              "  'You can find here the answers you need.'],\n",
              " array([0.6454993 , 0.35628742, 0.2875612 , 0.23560844, 0.21897565],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Ds7Sewi0UkK",
        "colab_type": "text"
      },
      "source": [
        "### Spanish to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jw6VKd5Q0RSC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Puede encontrar las respuestas que necesita aquí.\"], model_file=es_model_file, add_bos=True, add_eos=True)[0] + es_offset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l6q6F4gN0Xdk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    tarf=tf.constant([1.0]),\n",
        "    bos_id=en_offset + 1,\n",
        "    eos_id=en_offset + 2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HqKnIckN0YpJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z0lkRUXn0Z6j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ENGbZPTv0bEe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "471a0f94-96f6-441e-a2a6-9afb173ef155"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=en_model_file))], probs_[0]"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['You can find the answers you need here.',\n",
              "  'You can find the answers you need.',\n",
              "  'You can find the answers that you need here.',\n",
              "  'You can find the answers that you need.',\n",
              "  'You can find the answers we need here.'],\n",
              " array([0.77349657, 0.40411666, 0.39717805, 0.27532604, 0.23908341],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zWZNlRxr0h02",
        "colab_type": "text"
      },
      "source": [
        "### Italian to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTxXW2ZA0cOO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Puoi trovare le risposte che cerchi qui.\"], model_file=it_model_file, add_bos=True, add_eos=True)[0] + it_offset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lu_Wkq4S0nai",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    tarf=tf.constant([1.0]),\n",
        "    bos_id=en_offset + 1,\n",
        "    eos_id=en_offset + 2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOP3axX70q4R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hU6gVsog0st5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7QPV1j3B0uEt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "2448604c-ca58-4e67-da6a-86d369be5b3f"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=en_model_file))], probs_[0]"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['You can find the answers you want.',\n",
              "  'You can find the courses you want here.',\n",
              "  'You can find the answers you want here.',\n",
              "  'You can find the stipulating answers.',\n",
              "  'You can find the courses you want.'],\n",
              " array([0.4607614 , 0.26085025, 0.24746253, 0.22695974, 0.19222859],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhFGv8de0vHg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess.close()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}