global_enc_dec_with_BahdanauAttention(Concat Alignment Model).ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mejJonsu2o7r"
      },
      "source": [
        "#### Taken from [tensorflow-tutorial ](https://www.tensorflow.org/tutorials/text/nmt_with_attention ) and added a *translate_batch()* function to translate a batch and dump outputs into a file"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "J0Qjg6vuaHNt"
      },
      "source": [
        "# Neural machine translation with attention"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "tnxXKDjq3jEL"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From d:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import unicodedata\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import io\n",
        "import time\n",
        "from dataset import NMTDataset\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "YsrDDMlKA2nd",
        "outputId": "d31e50ca-bb4b-4c79-e920-7d30dac76748"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "'ls' is not recognized as an internal or external command,\n",
            "operable program or batch file.\n"
          ]
        }
      ],
      "source": [
        "!ls utils\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GOi42V79Ydlr"
      },
      "source": [
        "We'll use the same dataset we worked on notebook-1 (text-processing). For our convenience we've created a utils/dataset.py file which returns train and validation tf.data.Dataset objects."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "sy9mcFmxBP4J",
        "outputId": "a5f3a663-71c5-4d97-acd2-cc9249042d77"
      },
      "outputs": [],
      "source": [
        "BUFFER_SIZE = 3200\n",
        "BATCH_SIZE = 32\n",
        "num_examples = 3000\n",
        "\n",
        "dataset_creator = NMTDataset('en-spa')\n",
        "train_dataset, val_dataset, inp_lang, targ_lang = dataset_creator.call(num_examples, BUFFER_SIZE, BATCH_SIZE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'<OOV>': 1,\n",
              " 'the': 2,\n",
              " 'of': 3,\n",
              " 'and': 4,\n",
              " 'in': 5,\n",
              " 'to': 6,\n",
              " 'a': 7,\n",
              " 'is': 8,\n",
              " 'for': 9,\n",
              " '<start>the': 10,\n",
              " 'on': 11,\n",
              " 'that': 12,\n",
              " 'with': 13,\n",
              " 'has': 14,\n",
              " 'be': 15,\n",
              " 'was': 16,\n",
              " 'will': 17,\n",
              " 'have': 18,\n",
              " 'as': 19,\n",
              " 'by': 20,\n",
              " 'are': 21,\n",
              " ',': 22,\n",
              " 'not': 23,\n",
              " 'from': 24,\n",
              " 'at': 25,\n",
              " 'this': 26,\n",
              " 'also': 27,\n",
              " 'been': 28,\n",
              " 'he': 29,\n",
              " 'minister': 30,\n",
              " 'his': 31,\n",
              " 'it': 32,\n",
              " 'said': 33,\n",
              " 'you': 34,\n",
              " 'an': 35,\n",
              " '.': 36,\n",
              " 'india': 37,\n",
              " '<start>he': 38,\n",
              " 'were': 39,\n",
              " 'their': 40,\n",
              " 'they': 41,\n",
              " 'government': 42,\n",
              " 'which': 43,\n",
              " 'all': 44,\n",
              " '.<end>': 45,\n",
              " 'or': 46,\n",
              " 'had': 47,\n",
              " 'who': 48,\n",
              " 'we': 49,\n",
              " 'indian': 50,\n",
              " '<start>this': 51,\n",
              " 'police': 52,\n",
              " 'i': 53,\n",
              " '<start>in': 54,\n",
              " 'state': 55,\n",
              " 'chief': 56,\n",
              " 'but': 57,\n",
              " 'new': 58,\n",
              " 'after': 59,\n",
              " 'one': 60,\n",
              " 'no': 61,\n",
              " 'over': 62,\n",
              " 'other': 63,\n",
              " 'against': 64,\n",
              " 'there': 65,\n",
              " 'him': 66,\n",
              " 'can': 67,\n",
              " 'about': 68,\n",
              " '<start>it': 69,\n",
              " 'would': 70,\n",
              " '<start>i': 71,\n",
              " 'its': 72,\n",
              " 'when': 73,\n",
              " 'up': 74,\n",
              " 'people': 75,\n",
              " 'per': 76,\n",
              " 'first': 77,\n",
              " 'them': 78,\n",
              " 'any': 79,\n",
              " '<start>we': 80,\n",
              " 'two': 81,\n",
              " 'rs': 82,\n",
              " 'such': 83,\n",
              " 'out': 84,\n",
              " 'film': 85,\n",
              " 'your': 86,\n",
              " 'said.<end>': 87,\n",
              " 'made': 88,\n",
              " 'under': 89,\n",
              " 'singh': 90,\n",
              " 'should': 91,\n",
              " 'president': 92,\n",
              " 'court': 93,\n",
              " 'more': 94,\n",
              " 'bjp': 95,\n",
              " '<start>a': 96,\n",
              " 'into': 97,\n",
              " 'time': 98,\n",
              " 'our': 99,\n",
              " 'being': 100,\n",
              " 'delhi': 101,\n",
              " 'what': 102,\n",
              " 'her': 103,\n",
              " 'case': 104,\n",
              " 'do': 105,\n",
              " 'these': 106,\n",
              " '<start>but': 107,\n",
              " 'only': 108,\n",
              " 'like': 109,\n",
              " 'my': 110,\n",
              " 'congress': 111,\n",
              " 'while': 112,\n",
              " 'taken': 113,\n",
              " 'than': 114,\n",
              " 'during': 115,\n",
              " 'if': 116,\n",
              " 'prime': 117,\n",
              " 'so': 118,\n",
              " 'make': 119,\n",
              " 'take': 120,\n",
              " 'very': 121,\n",
              " 'modi': 122,\n",
              " 'those': 123,\n",
              " 'shall': 124,\n",
              " '<start>however,': 125,\n",
              " '<start>they': 126,\n",
              " '<start>and': 127,\n",
              " 'day': 128,\n",
              " 'through': 129,\n",
              " 'development': 130,\n",
              " 'us': 131,\n",
              " 'she': 132,\n",
              " 'national': 133,\n",
              " 'most': 134,\n",
              " 'may': 135,\n",
              " '<start>there': 136,\n",
              " 'where': 137,\n",
              " 'held': 138,\n",
              " 'three': 139,\n",
              " 'part': 140,\n",
              " 'public': 141,\n",
              " 'high': 142,\n",
              " 'last': 143,\n",
              " 'get': 144,\n",
              " 'union': 145,\n",
              " 'many': 146,\n",
              " 'party': 147,\n",
              " 'year': 148,\n",
              " 'then': 149,\n",
              " 'how': 150,\n",
              " 'former': 151,\n",
              " 'did': 152,\n",
              " 'th': 153,\n",
              " 'between': 154,\n",
              " 'said,': 155,\n",
              " 'narendra': 156,\n",
              " 'cent': 157,\n",
              " 'central': 158,\n",
              " 'must': 159,\n",
              " 'some': 160,\n",
              " 'including': 161,\n",
              " 'international': 162,\n",
              " 'social': 163,\n",
              " 'ministry': 164,\n",
              " 'good': 165,\n",
              " 'work': 166,\n",
              " 'before': 167,\n",
              " 'women': 168,\n",
              " 'it.<end>': 169,\n",
              " 'lakh': 170,\n",
              " 'committee': 171,\n",
              " 'allah': 172,\n",
              " 'need': 173,\n",
              " 'accused': 174,\n",
              " 'department': 175,\n",
              " 'production': 176,\n",
              " 'second': 177,\n",
              " 'god': 178,\n",
              " 'secretary': 179,\n",
              " 'could': 180,\n",
              " 'does': 181,\n",
              " '<start>she': 182,\n",
              " 'order': 183,\n",
              " 'world': 184,\n",
              " 'registered': 185,\n",
              " 'home': 186,\n",
              " 'due': 187,\n",
              " 'set': 188,\n",
              " 'support': 189,\n",
              " 'life': 190,\n",
              " '<start>as': 191,\n",
              " 'video': 192,\n",
              " 'great': 193,\n",
              " 'become': 194,\n",
              " 'years': 195,\n",
              " 'foreign': 196,\n",
              " '<start>all': 197,\n",
              " '<start>for': 198,\n",
              " 'crore': 199,\n",
              " 'official': 200,\n",
              " '<start>india': 201,\n",
              " 'leader': 202,\n",
              " 'sent': 203,\n",
              " 'help': 204,\n",
              " 'india.<end>': 205,\n",
              " 'place': 206,\n",
              " 'power': 207,\n",
              " 'shri': 208,\n",
              " 'given': 209,\n",
              " 'even': 210,\n",
              " 'country': 211,\n",
              " 'gandhi': 212,\n",
              " 'water': 213,\n",
              " 'air': 214,\n",
              " 'states': 215,\n",
              " 'kumar': 216,\n",
              " 'every': 217,\n",
              " 'team': 218,\n",
              " 'among': 219,\n",
              " 'come': 220,\n",
              " 'just': 221,\n",
              " 'general': 222,\n",
              " 'pradesh': 223,\n",
              " '<start>on': 224,\n",
              " 'country.<end>': 225,\n",
              " 'india,': 226,\n",
              " 'special': 227,\n",
              " 'media': 228,\n",
              " 'meeting': 229,\n",
              " 'family': 230,\n",
              " 'see': 231,\n",
              " 'five': 232,\n",
              " 'today': 233,\n",
              " 'cases': 234,\n",
              " 'know': 235,\n",
              " 'started': 236,\n",
              " '<start>what': 237,\n",
              " 'along': 238,\n",
              " 'four': 239,\n",
              " 'off': 240,\n",
              " 'number': 241,\n",
              " 'across': 242,\n",
              " 'district': 243,\n",
              " 'same': 244,\n",
              " 'north': 245,\n",
              " 'west': 246,\n",
              " 'because': 247,\n",
              " 'came': 248,\n",
              " 'sector': 249,\n",
              " 'state.<end>': 250,\n",
              " 'face': 251,\n",
              " 'important': 252,\n",
              " 'covid': 253,\n",
              " 'me': 254,\n",
              " 'present': 255,\n",
              " 'got': 256,\n",
              " 'action': 257,\n",
              " 'members': 258,\n",
              " 'back': 259,\n",
              " 's': 260,\n",
              " 'health': 261,\n",
              " 'role': 262,\n",
              " 'different': 263,\n",
              " 'countries': 264,\n",
              " '<start>police': 265,\n",
              " 'go': 266,\n",
              " 'section': 267,\n",
              " 'old': 268,\n",
              " 'arrested': 269,\n",
              " 'political': 270,\n",
              " 'kapoor': 271,\n",
              " 'think': 272,\n",
              " 'rahul': 273,\n",
              " 'around': 274,\n",
              " 'students': 275,\n",
              " 'services': 276,\n",
              " 'bank': 277,\n",
              " 'want': 278,\n",
              " 'both': 279,\n",
              " 'university': 280,\n",
              " '<start>prime': 281,\n",
              " 'well': 282,\n",
              " 'total': 283,\n",
              " 'school': 284,\n",
              " 'now': 285,\n",
              " 'candidates': 286,\n",
              " '<start>you': 287,\n",
              " 'took': 288,\n",
              " 'mp': 289,\n",
              " 'khan': 290,\n",
              " 'industry': 291,\n",
              " 'law': 292,\n",
              " 'system': 293,\n",
              " 'them.<end>': 294,\n",
              " 'says': 295,\n",
              " 'cannot': 296,\n",
              " \"'s\": 297,\n",
              " 'farmers': 298,\n",
              " 'till': 299,\n",
              " 'without': 300,\n",
              " 'persons': 301,\n",
              " 'post': 302,\n",
              " 'asked': 303,\n",
              " 'free': 304,\n",
              " 'done': 305,\n",
              " 'sabha': 306,\n",
              " 'local': 307,\n",
              " 'united': 308,\n",
              " 'lord': 309,\n",
              " 'days': 310,\n",
              " 'full': 311,\n",
              " '<start>so': 312,\n",
              " 'died': 313,\n",
              " 'business': 314,\n",
              " 'matter': 315,\n",
              " '<start>if': 316,\n",
              " 'called': 317,\n",
              " 'used': 318,\n",
              " 'best': 319,\n",
              " 'say': 320,\n",
              " 'shared': 321,\n",
              " 'seen': 322,\n",
              " 'going': 323,\n",
              " 'way': 324,\n",
              " 'test': 325,\n",
              " 'never': 326,\n",
              " 'community': 327,\n",
              " 'saying': 328,\n",
              " 'currently': 329,\n",
              " 'security': 330,\n",
              " 'captain': 331,\n",
              " 'justice': 332,\n",
              " 'technology': 333,\n",
              " 'upon': 334,\n",
              " 'name': 335,\n",
              " 'results': 336,\n",
              " 'election': 337,\n",
              " 'sharma': 338,\n",
              " 'assembly': 339,\n",
              " 'much': 340,\n",
              " 'people.<end>': 341,\n",
              " 'various': 342,\n",
              " 'further': 343,\n",
              " 'information': 344,\n",
              " 'am': 345,\n",
              " 'already': 346,\n",
              " 'result': 347,\n",
              " 'received': 348,\n",
              " 'children': 349,\n",
              " 'this,': 350,\n",
              " 'programme': 351,\n",
              " 'group': 352,\n",
              " 'son': 353,\n",
              " 'leaders': 354,\n",
              " 'officials': 355,\n",
              " 'issue': 356,\n",
              " 'small': 357,\n",
              " 'provide': 358,\n",
              " 'price': 359,\n",
              " 'control': 360,\n",
              " 'god,': 361,\n",
              " 'defence': 362,\n",
              " 'projects': 363,\n",
              " 'own': 364,\n",
              " 'minister,': 365,\n",
              " 'man': 366,\n",
              " 'next': 367,\n",
              " 'pakistan': 368,\n",
              " 'data': 369,\n",
              " 'area': 370,\n",
              " 'bengal': 371,\n",
              " 'known': 372,\n",
              " 'based': 373,\n",
              " 'ministers': 374,\n",
              " 'senior': 375,\n",
              " 'gods': 376,\n",
              " 'him.<end>': 377,\n",
              " 'rs.': 378,\n",
              " '<start>when': 379,\n",
              " 'won': 380,\n",
              " 'several': 381,\n",
              " 'reached': 382,\n",
              " 'spot.<end>': 383,\n",
              " 'process': 384,\n",
              " 'coming': 385,\n",
              " 'value': 386,\n",
              " '<start>after': 387,\n",
              " 'person': 388,\n",
              " '<start>new': 389,\n",
              " 'each': 390,\n",
              " 'long': 391,\n",
              " 'always': 392,\n",
              " 'look': 393,\n",
              " 'pm': 394,\n",
              " 'energy': 395,\n",
              " 'policy': 396,\n",
              " 'away': 397,\n",
              " 'government.<end>': 398,\n",
              " 'young': 399,\n",
              " 'police.<end>': 400,\n",
              " 'use': 401,\n",
              " 'return': 402,\n",
              " 'put': 403,\n",
              " 'remain': 404,\n",
              " '<start>his': 405,\n",
              " '<start>at': 406,\n",
              " 'decision': 407,\n",
              " 'actor': 408,\n",
              " 'related': 409,\n",
              " 'change': 410,\n",
              " 'interest': 411,\n",
              " 'available': 412,\n",
              " 'shah': 413,\n",
              " '<start>how': 414,\n",
              " '<start>to': 415,\n",
              " 'least': 416,\n",
              " 'daughter': 417,\n",
              " 'short': 418,\n",
              " 'released': 419,\n",
              " 'mla': 420,\n",
              " 'cooperation': 421,\n",
              " 'awarded': 422,\n",
              " 'nation': 423,\n",
              " 'this.<end>': 424,\n",
              " 'taking': 425,\n",
              " 'authority': 426,\n",
              " 'hold': 427,\n",
              " 'right': 428,\n",
              " 'might': 429,\n",
              " 'civil': 430,\n",
              " 'welfare': 431,\n",
              " 'event': 432,\n",
              " 'medical': 433,\n",
              " 'far': 434,\n",
              " 'governor': 435,\n",
              " 'member': 436,\n",
              " 'strong': 437,\n",
              " 'financial': 438,\n",
              " 'still': 439,\n",
              " 'brought': 440,\n",
              " '<start>during': 441,\n",
              " 'them,': 442,\n",
              " '<start>people': 443,\n",
              " 'release': 444,\n",
              " 'increase': 445,\n",
              " 'state,': 446,\n",
              " 'officers': 447,\n",
              " 'house': 448,\n",
              " 'following': 449,\n",
              " 'governments': 450,\n",
              " 'death': 451,\n",
              " 'making': 452,\n",
              " 'provided': 453,\n",
              " 'efforts': 454,\n",
              " 'why': 455,\n",
              " 'give': 456,\n",
              " 'situation': 457,\n",
              " 'cricket': 458,\n",
              " 'service': 459,\n",
              " 'account': 460,\n",
              " 'bring': 461,\n",
              " 'wife': 462,\n",
              " 'large': 463,\n",
              " 'youth': 464,\n",
              " 'providing': 465,\n",
              " 'clear': 466,\n",
              " 'decided': 467,\n",
              " 'occasion': 468,\n",
              " 'madhya': 469,\n",
              " 'complaint': 470,\n",
              " 'keep': 471,\n",
              " 'body': 472,\n",
              " 'jesus': 473,\n",
              " 'temple': 474,\n",
              " 'end': 475,\n",
              " 'mt': 476,\n",
              " 'having': 477,\n",
              " 'me.<end>': 478,\n",
              " 'include': 479,\n",
              " 'ram': 480,\n",
              " 'areas': 481,\n",
              " 'force': 482,\n",
              " 'anil': 483,\n",
              " 'meet': 484,\n",
              " 'major': 485,\n",
              " '<start>meanwhile,': 486,\n",
              " 'told': 487,\n",
              " 'ever': 488,\n",
              " 'space': 489,\n",
              " 'vice': 490,\n",
              " 'working': 491,\n",
              " 'capital': 492,\n",
              " 'private': 493,\n",
              " 'commission': 494,\n",
              " 'report': 495,\n",
              " 'responsibility': 496,\n",
              " 'view': 497,\n",
              " 'woman': 498,\n",
              " 'didnt': 499,\n",
              " 'create': 500,\n",
              " 'higher': 501,\n",
              " 'management': 502,\n",
              " 'jehovah': 503,\n",
              " 'able': 504,\n",
              " 'series': 505,\n",
              " 'cabinet': 506,\n",
              " 'love': 507,\n",
              " 'single': 508,\n",
              " 'board': 509,\n",
              " 'regard.<end>': 510,\n",
              " 'tax': 511,\n",
              " 'needs': 512,\n",
              " 'land': 513,\n",
              " '<start>its': 514,\n",
              " 'there.<end>': 515,\n",
              " 'spot': 516,\n",
              " 'jammu': 517,\n",
              " 'worked': 518,\n",
              " 'centre': 519,\n",
              " 'phone': 520,\n",
              " 'peace': 521,\n",
              " 'festival': 522,\n",
              " 'punjab': 523,\n",
              " 'named': 524,\n",
              " 'sena': 525,\n",
              " 'later': 526,\n",
              " 'fast': 527,\n",
              " 'continue': 528,\n",
              " 'growth': 529,\n",
              " 'bilateral': 530,\n",
              " 'food': 531,\n",
              " 'road': 532,\n",
              " 'houses': 533,\n",
              " 'increased': 534,\n",
              " 'plan': 535,\n",
              " 'fire': 536,\n",
              " 'september': 537,\n",
              " 'few': 538,\n",
              " 'oil': 539,\n",
              " 'believe': 540,\n",
              " 'final': 541,\n",
              " 'share': 542,\n",
              " 'word': 543,\n",
              " 'charge': 544,\n",
              " 'case.<end>': 545,\n",
              " 'hospital': 546,\n",
              " 'capacity': 547,\n",
              " 'killed': 548,\n",
              " 'act': 549,\n",
              " 'non': 550,\n",
              " 'supreme': 551,\n",
              " 'show': 552,\n",
              " 'company': 553,\n",
              " 'focus': 554,\n",
              " 'society': 555,\n",
              " 'father': 556,\n",
              " 'delhi.<end>': 557,\n",
              " 'deputy': 558,\n",
              " 'start': 559,\n",
              " 'found': 560,\n",
              " 'australia': 561,\n",
              " 'alleged': 562,\n",
              " 'past': 563,\n",
              " 'compared': 564,\n",
              " 'attended': 565,\n",
              " 'yadav': 566,\n",
              " 'k': 567,\n",
              " 'research': 568,\n",
              " 'month': 569,\n",
              " 'finance': 570,\n",
              " 'added': 571,\n",
              " 'form': 572,\n",
              " 'office': 573,\n",
              " 'faith': 574,\n",
              " 'let': 575,\n",
              " 'added.<end>': 576,\n",
              " 'institute': 577,\n",
              " 'education': 578,\n",
              " 'brother': 579,\n",
              " 'uttar': 580,\n",
              " 'early': 581,\n",
              " 'concerned': 582,\n",
              " 'income': 583,\n",
              " '<start>with': 584,\n",
              " 'year.<end>': 585,\n",
              " 'bollywood': 586,\n",
              " 'elections': 587,\n",
              " 'according': 588,\n",
              " 'ye': 589,\n",
              " 'speak': 590,\n",
              " 'class': 591,\n",
              " 'ensure': 592,\n",
              " '<start>according': 593,\n",
              " 'spirit': 594,\n",
              " 'match': 595,\n",
              " 'affairs': 596,\n",
              " 'however,': 597,\n",
              " 'down': 598,\n",
              " 'informed': 599,\n",
              " '<start>delhi': 600,\n",
              " 'failed': 601,\n",
              " 'true': 602,\n",
              " 'includes': 603,\n",
              " 'pradesh,': 604,\n",
              " '<start>former': 605,\n",
              " 'injured': 606,\n",
              " 'incident.<end>': 607,\n",
              " 'singh,': 608,\n",
              " 'investigation': 609,\n",
              " 'money': 610,\n",
              " 'read': 611,\n",
              " 'director': 612,\n",
              " 'million': 613,\n",
              " 'greater': 614,\n",
              " 'east': 615,\n",
              " 'them.': 616,\n",
              " 'whom': 617,\n",
              " 'commerce': 618,\n",
              " 'since': 619,\n",
              " 'written': 620,\n",
              " 'visit': 621,\n",
              " 'economic': 622,\n",
              " 'december': 623,\n",
              " 'book': 624,\n",
              " 'rural': 625,\n",
              " 'association': 626,\n",
              " 'top': 627,\n",
              " 'organised': 628,\n",
              " 'knowledge': 629,\n",
              " 'market': 630,\n",
              " 'announced': 631,\n",
              " 'rules': 632,\n",
              " 'times': 633,\n",
              " 'living': 634,\n",
              " 'village': 635,\n",
              " 'themselves': 636,\n",
              " 'gujarat': 637,\n",
              " 'railways': 638,\n",
              " 'organisation': 639,\n",
              " 'committed': 640,\n",
              " 'towards': 641,\n",
              " 'looking': 642,\n",
              " 'protection': 643,\n",
              " 'issued': 644,\n",
              " 'red': 645,\n",
              " 'muslim': 646,\n",
              " 'earlier': 647,\n",
              " 'matter.<end>': 648,\n",
              " 'cost': 649,\n",
              " 'stay': 650,\n",
              " '<start>why': 651,\n",
              " 'him,': 652,\n",
              " 'here': 653,\n",
              " 'became': 654,\n",
              " 'comes': 655,\n",
              " 'south': 656,\n",
              " 'lok': 657,\n",
              " 'college': 658,\n",
              " '<start>no': 659,\n",
              " 'lot': 660,\n",
              " '<start>one': 661,\n",
              " 'entire': 662,\n",
              " 'upcoming': 663,\n",
              " 'until': 664,\n",
              " 'six': 665,\n",
              " 'kashmir': 666,\n",
              " 'active': 667,\n",
              " 'virat': 668,\n",
              " 'cbi': 669,\n",
              " 'white': 670,\n",
              " 'amit': 671,\n",
              " '<start>two': 672,\n",
              " 'participated': 673,\n",
              " 'ahead': 674,\n",
              " 'parts': 675,\n",
              " 't': 676,\n",
              " 'website': 677,\n",
              " 'bihar': 678,\n",
              " 'within': 679,\n",
              " 'bharat': 680,\n",
              " 'regional': 681,\n",
              " 'industrial': 682,\n",
              " 'add': 683,\n",
              " 'regarding': 684,\n",
              " 'pay': 685,\n",
              " 'holy': 686,\n",
              " 'personal': 687,\n",
              " 'companies': 688,\n",
              " 'reported': 689,\n",
              " 'basis': 690,\n",
              " 'respect': 691,\n",
              " 'patel': 692,\n",
              " 'movement': 693,\n",
              " 'exchange': 694,\n",
              " 'opportunity': 695,\n",
              " 'demanded': 696,\n",
              " 'third': 697,\n",
              " 'worship': 698,\n",
              " 'nothing': 699,\n",
              " 'infrastructure': 700,\n",
              " 'gold': 701,\n",
              " 'likely': 702,\n",
              " 'statement.<end>': 703,\n",
              " 'something': 704,\n",
              " '<start>so,': 705,\n",
              " 'haryana,': 706,\n",
              " 'agreed': 707,\n",
              " 'story': 708,\n",
              " 'gives': 709,\n",
              " 'that,': 710,\n",
              " '<start>pm': 711,\n",
              " '<start>that': 712,\n",
              " 'girl': 713,\n",
              " 'media.<end>': 714,\n",
              " 'railway': 715,\n",
              " 'england': 716,\n",
              " 'call': 717,\n",
              " 'gb': 718,\n",
              " 'hard': 719,\n",
              " 'radio': 720,\n",
              " 'real': 721,\n",
              " 'phase': 722,\n",
              " 'affected': 723,\n",
              " 'region': 724,\n",
              " 'council': 725,\n",
              " 'june': 726,\n",
              " 'run': 727,\n",
              " 'whether': 728,\n",
              " '<start>union': 729,\n",
              " 'played': 730,\n",
              " 'army': 731,\n",
              " 'cm': 732,\n",
              " 'goods': 733,\n",
              " 'stop': 734,\n",
              " 'especially': 735,\n",
              " 'whose': 736,\n",
              " 'study': 737,\n",
              " 'bible': 738,\n",
              " 'kmph': 739,\n",
              " 'd': 740,\n",
              " 'western': 741,\n",
              " 'response': 742,\n",
              " 'caused': 743,\n",
              " 'incident': 744,\n",
              " 'stated': 745,\n",
              " 'led': 746,\n",
              " 'low': 747,\n",
              " 'produced': 748,\n",
              " 'personnel': 749,\n",
              " 'bench': 750,\n",
              " 'main': 751,\n",
              " 'prices': 752,\n",
              " 'karnataka': 753,\n",
              " 'together': 754,\n",
              " 'stand': 755,\n",
              " 'truth': 756,\n",
              " 'popular': 757,\n",
              " 'additional': 758,\n",
              " 'feel': 759,\n",
              " 'investment': 760,\n",
              " 'similar': 761,\n",
              " '<start>by': 762,\n",
              " 'states.<end>': 763,\n",
              " 'near': 764,\n",
              " 'her.<end>': 765,\n",
              " 'rather': 766,\n",
              " 'rajya': 767,\n",
              " 'out.<end>': 768,\n",
              " 'e': 769,\n",
              " 'developed': 770,\n",
              " 'implementation': 771,\n",
              " 'measures': 772,\n",
              " 'kohli': 773,\n",
              " 'located': 774,\n",
              " 'moon': 775,\n",
              " 'head': 776,\n",
              " 'understand': 777,\n",
              " 'forms': 778,\n",
              " 'zealand': 779,\n",
              " 'bharatiya': 780,\n",
              " 'project': 781,\n",
              " '<start>similarly,': 782,\n",
              " 'article': 783,\n",
              " 'child': 784,\n",
              " 'expressed': 785,\n",
              " 'send': 786,\n",
              " 'claimed': 787,\n",
              " 'employment': 788,\n",
              " 'attack': 789,\n",
              " 'indias': 790,\n",
              " 'sub': 791,\n",
              " 'human': 792,\n",
              " 'al': 793,\n",
              " 'well.<end>': 794,\n",
              " 'km': 795,\n",
              " 'celebrated': 796,\n",
              " 'agreement': 797,\n",
              " 'performed': 798,\n",
              " 'co': 799,\n",
              " 'cultural': 800,\n",
              " 'everyone': 801,\n",
              " 'seats': 802,\n",
              " 'behind': 803,\n",
              " 'janata': 804,\n",
              " 'soon.<end>': 805,\n",
              " 'wicket': 806,\n",
              " 'transport': 807,\n",
              " 'reach': 808,\n",
              " 'joint': 809,\n",
              " 'august': 810,\n",
              " 'reduce': 811,\n",
              " 'that.<end>': 812,\n",
              " 'experience': 813,\n",
              " 'included': 814,\n",
              " 'opened': 815,\n",
              " 'female': 816,\n",
              " 'identified': 817,\n",
              " 'rukh': 818,\n",
              " 'anything': 819,\n",
              " 'strike': 820,\n",
              " 'country,': 821,\n",
              " 'happened': 822,\n",
              " 'it.': 823,\n",
              " 'heard': 824,\n",
              " 'mentioned': 825,\n",
              " 'heart': 826,\n",
              " 'electric': 827,\n",
              " 'published': 828,\n",
              " 'schemes': 829,\n",
              " 'black': 830,\n",
              " 'either': 831,\n",
              " 'storage': 832,\n",
              " '<start>chief': 833,\n",
              " 'quite': 834,\n",
              " 'ali': 835,\n",
              " 'review': 836,\n",
              " 'band': 837,\n",
              " 'above': 838,\n",
              " 'previous': 839,\n",
              " 'affairs,': 840,\n",
              " '<start>apart': 841,\n",
              " 'filed': 842,\n",
              " 'gave': 843,\n",
              " 'provisions': 844,\n",
              " 'movie': 845,\n",
              " 'means': 846,\n",
              " 'yet': 847,\n",
              " 'husband': 848,\n",
              " 'monitoring': 849,\n",
              " 'key': 850,\n",
              " 'driver': 851,\n",
              " 'trade': 852,\n",
              " 'occasion.<end>': 853,\n",
              " 'districts': 854,\n",
              " 'city': 855,\n",
              " 'check': 856,\n",
              " 'china': 857,\n",
              " 'mind': 858,\n",
              " '<start>congress': 859,\n",
              " 'regular': 860,\n",
              " 'twenty': 861,\n",
              " 'directed': 862,\n",
              " 'bill': 863,\n",
              " 'hearing': 864,\n",
              " 'operation': 865,\n",
              " 'launched': 866,\n",
              " '<start>jesus': 867,\n",
              " 'parties': 868,\n",
              " 'week': 869,\n",
              " 'dont': 870,\n",
              " 'allow': 871,\n",
              " 'sp': 872,\n",
              " '<start>some': 873,\n",
              " 'mumbai': 874,\n",
              " '<start>both': 875,\n",
              " 'appointed': 876,\n",
              " 'raised': 877,\n",
              " 'terrorism': 878,\n",
              " 'submitted': 879,\n",
              " 'thought': 880,\n",
              " 'today.<end>': 881,\n",
              " 'issues': 882,\n",
              " 'patients': 883,\n",
              " 'sad': 884,\n",
              " 'extended': 885,\n",
              " 'season': 886,\n",
              " 'network': 887,\n",
              " 'date': 888,\n",
              " '<start>following': 889,\n",
              " 'lower': 890,\n",
              " 'facing': 891,\n",
              " 'matters': 892,\n",
              " 'difficult': 893,\n",
              " 'up.<end>': 894,\n",
              " 'provides': 895,\n",
              " 'cover': 896,\n",
              " 'nuclear': 897,\n",
              " 'connection': 898,\n",
              " 'world.<end>': 899,\n",
              " 'seek': 900,\n",
              " 'provision': 901,\n",
              " 'list': 902,\n",
              " 'iphone': 903,\n",
              " 'question': 904,\n",
              " 'live': 905,\n",
              " 'time,': 906,\n",
              " 'holding': 907,\n",
              " 'candidate': 908,\n",
              " 'song': 909,\n",
              " '<start>mumbai': 910,\n",
              " 'wrote': 911,\n",
              " 'quoted': 912,\n",
              " 'nation.<end>': 913,\n",
              " 'conducted': 914,\n",
              " 'record': 915,\n",
              " 'except': 916,\n",
              " '<start>later,': 917,\n",
              " 'happen': 918,\n",
              " 'statement': 919,\n",
              " 'using': 920,\n",
              " 'rest': 921,\n",
              " 'workers': 922,\n",
              " 'effective': 923,\n",
              " 'care': 924,\n",
              " 'terms': 925,\n",
              " 'legislative': 926,\n",
              " 'win': 927,\n",
              " 'married': 928,\n",
              " 'kind': 929,\n",
              " 'benefit': 930,\n",
              " 'nature': 931,\n",
              " 'bjp.<end>': 932,\n",
              " 'de': 933,\n",
              " 'offered': 934,\n",
              " 'deal': 935,\n",
              " 'trying': 936,\n",
              " 'secretary,': 937,\n",
              " 'effect': 938,\n",
              " 'fall': 939,\n",
              " '<start>many': 940,\n",
              " 'stage': 941,\n",
              " 'hindi': 942,\n",
              " 'shiv': 943,\n",
              " 'getting': 944,\n",
              " 'import': 945,\n",
              " '<start>indian': 946,\n",
              " 'nehru': 947,\n",
              " '<start>ministry': 948,\n",
              " 'spread': 949,\n",
              " 'areas.<end>': 950,\n",
              " 'r': 951,\n",
              " 'try': 952,\n",
              " 'reports': 953,\n",
              " 'sitting': 954,\n",
              " '<start>students': 955,\n",
              " 'communication': 956,\n",
              " 'developing': 957,\n",
              " 'met': 958,\n",
              " 'tourism': 959,\n",
              " 'agricultural': 960,\n",
              " 'bodies': 961,\n",
              " 'reason': 962,\n",
              " 'commissioner': 963,\n",
              " 'century': 964,\n",
              " 'c': 965,\n",
              " 'term': 966,\n",
              " 'aap': 967,\n",
              " 'court.<end>': 968,\n",
              " 'players': 969,\n",
              " 'history': 970,\n",
              " 'pakistani': 971,\n",
              " 'domestic': 972,\n",
              " 'throughout': 973,\n",
              " 'institutions': 974,\n",
              " 'current': 975,\n",
              " 'powered': 976,\n",
              " 'level': 977,\n",
              " '<start>not': 978,\n",
              " 'seeking': 979,\n",
              " 'population': 980,\n",
              " 'questions': 981,\n",
              " 'time.<end>': 982,\n",
              " 'really': 983,\n",
              " 'happy': 984,\n",
              " 'metric': 985,\n",
              " '<start>earlier,': 986,\n",
              " 'tennis': 987,\n",
              " 'gas': 988,\n",
              " 'january': 989,\n",
              " 'demand': 990,\n",
              " '<start>bjp': 991,\n",
              " 'enter': 992,\n",
              " 'year,': 993,\n",
              " 'saw': 994,\n",
              " 'often': 995,\n",
              " 'signed': 996,\n",
              " 'legal': 997,\n",
              " 'integral': 998,\n",
              " 'light': 999,\n",
              " 'conditions': 1000,\n",
              " ...}"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "inp_lang.word_index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "wVtSz89FBk0D",
        "outputId": "48d1ab71-9d23-4369-f081-12c8927814c2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Inpute Vocabulary Size: 12545\n",
            "Target Vocabulary Size: 10656\n"
          ]
        }
      ],
      "source": [
        "\n",
        "print(\"Inpute Vocabulary Size: {}\".format(len(inp_lang.word_index)))\n",
        "print(\"Target Vocabulary Size: {}\".format(len(targ_lang.word_index)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "qc6-NK1GtWQt",
        "outputId": "f35ff927-5a3c-480c-95ae-2b0d63844250"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(TensorShape([32, 128]), TensorShape([32, 167]))"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "example_input_batch, example_target_batch = next(iter(train_dataset))\n",
        "example_input_batch.shape, example_target_batch.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "TNfHIF71ulLu"
      },
      "source": [
        "## Write the encoder and decoder model\n",
        "\n",
        "Implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://github.com/tensorflow/nmt). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://github.com/tensorflow/nmt#background-on-the-attention-mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence. The below picture and formulas are an example of attention mechanism from [Luong's paper](https://arxiv.org/abs/1508.04025v5). \n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\">\n",
        "\n",
        "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*.\n",
        "\n",
        "Here are the equations that are implemented:\n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\">\n",
        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\">\n",
        "\n",
        "This tutorial uses [Bahdanau attention](https://arxiv.org/pdf/1409.0473.pdf) for the encoder. Let's decide on notation before writing the simplified form:\n",
        "\n",
        "* FC = Fully connected (dense) layer\n",
        "* EO = Encoder output\n",
        "* H = hidden state\n",
        "* X = input to the decoder\n",
        "\n",
        "And the pseudo-code:\n",
        "\n",
        "* `score = FC(tanh(FC(EO) + FC(H)))`\n",
        "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, hidden_size)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
        "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
        "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
        "* `merged vector = concat(embedding output, context vector)`\n",
        "* This merged vector is then given to the GRU\n",
        "\n",
        "The shapes of all the vectors at each step have been specified in the comments in the code:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "dgVsBjbVDBwX"
      },
      "outputs": [],
      "source": [
        "# Define some useful parameters for further use\n",
        "\n",
        "vocab_inp_size = len(inp_lang.word_index)+1\n",
        "vocab_tar_size = len(targ_lang.word_index)+1\n",
        "max_length_input = example_input_batch.shape[1]\n",
        "max_length_output = example_target_batch.shape[1]\n",
        "\n",
        "embedding_dim = 128\n",
        "units = 124\n",
        "steps_per_epoch = num_examples//BATCH_SIZE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(12546, 10657, 128, 167)"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "vocab_inp_size,vocab_tar_size,max_length_input,max_length_output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "nZ2rI24i3jFg"
      },
      "outputs": [],
      "source": [
        "# Encoder is composed of embedding layer and then one GRU layer. It produces outputs and last hidden states. \n",
        "# Encoder Outputs shape = (BATCH_SIZE, max_length_input, units)\n",
        "# Last Hidden State Shape = (BATCH_SIZE, units)\n",
        "\n",
        "class Encoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
        "    super().__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.enc_units = enc_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "\n",
        "  def call(self, x, hidden):\n",
        "    x = self.embedding(x)\n",
        "    print(f'needed {x.shape}')\n",
        "    output, state = self.gru(x, initial_state = hidden)\n",
        "    return output, state\n",
        "\n",
        "  def initialize_hidden_state(self):\n",
        "    return tf.zeros((self.batch_sz, self.enc_units))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "60gSVh05Jl6l",
        "outputId": "d5ad818c-8ef1-4f16-e9fb-b04503a5ca8a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From d:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\keras\\src\\backend.py:873: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "needed (32, 152, 128)\n",
            "Encoder output shape: (batch size, sequence length, units) (32, 152, 124)\n",
            "Encoder Hidden state shape: (batch size, units) (32, 124)\n"
          ]
        }
      ],
      "source": [
        "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
        "\n",
        "# sample input\n",
        "sample_hidden = encoder.initialize_hidden_state()\n",
        "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
        "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
        "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "umohpBN2OM94"
      },
      "outputs": [],
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, units):\n",
        "    super(BahdanauAttention, self).__init__()\n",
        "    # To recall, score = V*tanh(W1(encoder_outputs) + W2(Prev Step's Hidden State))\n",
        "    self.W1 = tf.keras.layers.Dense(units)\n",
        "    self.W2 = tf.keras.layers.Dense(units)\n",
        "    self.V = tf.keras.layers.Dense(1)\n",
        "\n",
        "  def call(self, query, values):\n",
        "    # query hidden state shape == (batch_size, hidden size) mine (m,n_s)\n",
        "    # query_with_time_axis shape == (batch_size, 1, hidden size) mine (m,1,n_s)\n",
        "    # values shape == (batch_size, max_len, hidden size) mine (m,maxLen,n_a)\n",
        "    # we are doing this to broadcast addition along the time axis to calculate the score\n",
        "    query_with_time_axis = tf.expand_dims(query, 1) ## (m,1,n_s)\n",
        "\n",
        "    # score shape == (batch_size, max_length, 1) mine (m,maxLen,1)\n",
        "    # we get 1 at the last axis because we are applying score to self.V\n",
        "    # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
        "    \n",
        "    #-------- COMPUTING EQUATION (4) (Concat Alignment Model) ---------#  \n",
        "    score = self.V(tf.nn.tanh(\n",
        "        self.W1(query_with_time_axis) + self.W2(values))) ## shape is (m,maxLen,1)\n",
        "\n",
        "    #-------- COMPUTING EQUATION (1) ------------#\n",
        "    # attention_weights shape == (batch_size, max_length, 1) \n",
        "    attention_weights = tf.nn.softmax(score, axis=1) ## shape is (m,maxlen,1) \n",
        "\n",
        "    #---------- COMPUTING EQUATION (2) -----------#\n",
        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
        "    context_vector = attention_weights * values ## shape is (m,maxLen,n_a)\n",
        "    # Context vector is passed on to the curren time step's decoder cell\n",
        "    context_vector = tf.reduce_sum(context_vector, axis=1) # (m,n_a)\n",
        "\n",
        "    return context_vector, attention_weights"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "k534zTHiDjQU",
        "outputId": "6b662027-8c36-4798-f1a0-0f186f8dbc85"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Attention result shape: (batch size, units) (32, 124)\n",
            "Attention weights shape: (batch_size, sequence_length, 1) (32, 152, 1)\n"
          ]
        }
      ],
      "source": [
        "attention_layer = BahdanauAttention(10)\n",
        "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
        "\n",
        "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
        "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "yJ_B3mhW3jFk"
      },
      "outputs": [],
      "source": [
        "class Decoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
        "    super(Decoder, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.dec_units = dec_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "\n",
        "    # used for attention\n",
        "    self.attention = BahdanauAttention(self.dec_units)\n",
        "\n",
        "  def call(self, x, hidden, enc_output):\n",
        "    \n",
        "    # x is input of m trainable example at specific time step shape is (m,1)\n",
        "    \n",
        "    \n",
        "    # enc_output shape == (batch_size, max_length, hidden_size)\n",
        "    context_vector, attention_weights = self.attention(hidden, enc_output)\n",
        "\n",
        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "    x = self.embedding(x) # x shape is (m,1,emb_dim) (64,1,128)\n",
        "\n",
        "\n",
        "    #------------ COMPUTING EQUATION (3) ----------------------#\n",
        "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
        "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "    \n",
        "    # passing the concatenated vector to the GRU\n",
        "    output, state = self.gru(x) # (m,1,n_s)\n",
        "    # print(f'another needed {output.shape}')\n",
        "\n",
        "    # output shape == (batch_size * 1, hidden_size)\n",
        "    output = tf.reshape(output, (-1, output.shape[2])) # (m,n_s)\n",
        "\n",
        "    # output shape == (batch_size, vocab)\n",
        "    x = self.fc(output)  # (m,length_vocab)\n",
        "\n",
        "    return x, state, attention_weights"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "P5UY8wko3jFp",
        "outputId": "fee4a9eb-3040-47f7-b990-d3808e940516"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Decoder output shape: (batch_size, vocab size) (32, 9691)\n"
          ]
        }
      ],
      "source": [
        "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
        "\n",
        "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n",
        "                                      sample_hidden, sample_output)\n",
        "\n",
        "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_ch_71VbIRfK"
      },
      "source": [
        "## Define the optimizer and the loss function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "WmTHr5iV3jFr"
      },
      "outputs": [],
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')\n",
        "\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "\n",
        "  return tf.reduce_mean(loss_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "DMVWzzsfNl4e"
      },
      "source": [
        "## Checkpoints (Object-based saving)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "Zj8bXQTgNwrF"
      },
      "outputs": [],
      "source": [
        "checkpoint_dir = './training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
        "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
        "                                 encoder=encoder,\n",
        "                                 decoder=decoder)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hpObfY22IddU"
      },
      "source": [
        "## Training\n",
        "\n",
        "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n",
        "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n",
        "3. The decoder returns the *predictions* and the *decoder hidden state*.\n",
        "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
        "5. Use *teacher forcing* to decide the next input to the decoder.\n",
        "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n",
        "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "sC9ArXSsVfqn"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def train_step(inp, targ, enc_hidden): ## for single data batch training\n",
        "  loss = 0\n",
        "\n",
        "  with tf.GradientTape() as tape:\n",
        "    enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "    \n",
        "    # enc_hidden shape is (m,n_a)\n",
        "    # dec_hidden_shape is (m,n_s)\n",
        "    # Note n_a = n_s = 1024 \n",
        "    # dec hidden is initial hidden state of decoder and it got value of last hidden state of decoder \n",
        "    dec_hidden = enc_hidden\n",
        "\n",
        "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "    # Teacher forcing - feeding the target as the next input\n",
        "    for t in range(1, targ.shape[1]):\n",
        "      # passing enc_output to the decoder\n",
        "      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "\n",
        "      loss += loss_function(targ[:, t], predictions)\n",
        "\n",
        "      # using teacher forcing\n",
        "      dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "  batch_loss = (loss / int(targ.shape[1]))\n",
        "\n",
        "  variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "\n",
        "  gradients = tape.gradient(loss, variables)\n",
        "\n",
        "  optimizer.apply_gradients(zip(gradients, variables))\n",
        "\n",
        "  return batch_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "colab_type": "code",
        "id": "ddefjBMa3jF0",
        "outputId": "b9588938-68d5-4ac1-a2cd-d6f07d3d2b57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "needed (32, 152, 128)\n",
            "needed (32, 152, 128)\n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "Cell \u001b[1;32mIn[17], line 10\u001b[0m\n\u001b[0;32m      7\u001b[0m total_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (batch, (inp, targ)) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(train_dataset\u001b[38;5;241m.\u001b[39mtake(steps_per_epoch)):\n\u001b[1;32m---> 10\u001b[0m   batch_loss \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menc_hidden\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     11\u001b[0m   total_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m batch_loss\n\u001b[0;32m     13\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m batch \u001b[38;5;241m%\u001b[39m \u001b[38;5;241m100\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\util\\traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:832\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    829\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    831\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[1;32m--> 832\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[0;32m    834\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[0;32m    835\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:905\u001b[0m, in \u001b[0;36mFunction._call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    901\u001b[0m     \u001b[38;5;28;01mpass\u001b[39;00m  \u001b[38;5;66;03m# Fall through to cond-based initialization.\u001b[39;00m\n\u001b[0;32m    902\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    903\u001b[0m     \u001b[38;5;66;03m# Lifting succeeded, so variables are initialized and we can run the\u001b[39;00m\n\u001b[0;32m    904\u001b[0m     \u001b[38;5;66;03m# no_variable_creation function.\u001b[39;00m\n\u001b[1;32m--> 905\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtracing_compilation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    906\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_no_variable_creation_config\u001b[49m\n\u001b[0;32m    907\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    908\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    909\u001b[0m   bound_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_concrete_variable_creation_fn\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mbind(\n\u001b[0;32m    910\u001b[0m       \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds\n\u001b[0;32m    911\u001b[0m   )\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\tracing_compilation.py:139\u001b[0m, in \u001b[0;36mcall_function\u001b[1;34m(args, kwargs, tracing_options)\u001b[0m\n\u001b[0;32m    137\u001b[0m bound_args \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mbind(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    138\u001b[0m flat_inputs \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39munpack_inputs(bound_args)\n\u001b[1;32m--> 139\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# pylint: disable=protected-access\u001b[39;49;00m\n\u001b[0;32m    140\u001b[0m \u001b[43m    \u001b[49m\u001b[43mflat_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\n\u001b[0;32m    141\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\concrete_function.py:1323\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[1;34m(self, tensor_inputs, captured_inputs)\u001b[0m\n\u001b[0;32m   1319\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[0;32m   1320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[0;32m   1321\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[0;32m   1322\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[1;32m-> 1323\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_preflattened\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1324\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[0;32m   1325\u001b[0m     args,\n\u001b[0;32m   1326\u001b[0m     possible_gradient_type,\n\u001b[0;32m   1327\u001b[0m     executing_eagerly)\n\u001b[0;32m   1328\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:216\u001b[0m, in \u001b[0;36mAtomicFunction.call_preflattened\u001b[1;34m(self, args)\u001b[0m\n\u001b[0;32m    214\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall_preflattened\u001b[39m(\u001b[38;5;28mself\u001b[39m, args: Sequence[core\u001b[38;5;241m.\u001b[39mTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m    215\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"Calls with flattened tensor inputs and returns the structured output.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 216\u001b[0m   flat_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_flat\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    217\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mpack_output(flat_outputs)\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:251\u001b[0m, in \u001b[0;36mAtomicFunction.call_flat\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    249\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m record\u001b[38;5;241m.\u001b[39mstop_recording():\n\u001b[0;32m    250\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mexecuting_eagerly():\n\u001b[1;32m--> 251\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_bound_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    252\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    253\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    254\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mflat_outputs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    255\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    256\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    257\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m make_call_op_in_graph(\n\u001b[0;32m    258\u001b[0m         \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    259\u001b[0m         \u001b[38;5;28mlist\u001b[39m(args),\n\u001b[0;32m    260\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mfunction_call_options\u001b[38;5;241m.\u001b[39mas_attrs(),\n\u001b[0;32m    261\u001b[0m     )\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\context.py:1486\u001b[0m, in \u001b[0;36mContext.call_function\u001b[1;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[0;32m   1484\u001b[0m cancellation_context \u001b[38;5;241m=\u001b[39m cancellation\u001b[38;5;241m.\u001b[39mcontext()\n\u001b[0;32m   1485\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cancellation_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m-> 1486\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1487\u001b[0m \u001b[43m      \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1488\u001b[0m \u001b[43m      \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1489\u001b[0m \u001b[43m      \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtensor_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1490\u001b[0m \u001b[43m      \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1491\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1492\u001b[0m \u001b[43m  \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1493\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1494\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[0;32m   1495\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1496\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1500\u001b[0m       cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_context,\n\u001b[0;32m   1501\u001b[0m   )\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[1;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     54\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
            "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "EPOCHS = 3\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "\n",
        "  enc_hidden = encoder.initialize_hidden_state()\n",
        "  total_loss = 0\n",
        "\n",
        "  for (batch, (inp, targ)) in enumerate(train_dataset.take(steps_per_epoch)):\n",
        "    batch_loss = train_step(inp, targ, enc_hidden)\n",
        "    total_loss += batch_loss\n",
        "\n",
        "    if batch % 100 == 0:\n",
        "      print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                                   batch,\n",
        "                                                   batch_loss.numpy()))\n",
        "  # saving (checkpoint) the model every 2 epochs\n",
        "  if (epoch + 1) % 2 == 0:\n",
        "    checkpoint.save(file_prefix = checkpoint_prefix)\n",
        "\n",
        "  print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                      total_loss / steps_per_epoch))\n",
        "  print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mU3Ce8M6I3rz"
      },
      "source": [
        "## Translate\n",
        "\n",
        "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
        "* Stop predicting when the model predicts the *end token*.\n",
        "* And store the *attention weights for every time step*.\n",
        "\n",
        "Note: The encoder output is calculated only once for one input."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "jkk"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "EbQpyYs13jF_"
      },
      "outputs": [],
      "source": [
        "def evaluate(sentence):\n",
        "\n",
        "  sentence = dataset_creator.preprocess_sentence(sentence)\n",
        "  inputs = [sentence]\n",
        "  inputs = inp_lang.texts_to_sequences(inputs)\n",
        "  # inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
        "  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
        "                                                         maxlen=max_length_input,\n",
        "                                                         padding='post')\n",
        "  inputs = tf.convert_to_tensor(inputs)\n",
        "\n",
        "  result = ''\n",
        "\n",
        "  hidden = [tf.zeros((1, units))]\n",
        "  enc_out, enc_hidden = encoder(inputs, hidden)\n",
        "\n",
        "  dec_hidden = enc_hidden\n",
        "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
        "\n",
        "  for t in range(max_length_output):\n",
        "    predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
        "                                                         dec_hidden,\n",
        "                                                         enc_out)\n",
        "\n",
        "    # storing the attention weights to plot later on\n",
        "\n",
        "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "\n",
        "    result += targ_lang.index_word[predicted_id] + ' '\n",
        "\n",
        "    if targ_lang.index_word[predicted_id] == '<end>':\n",
        "      return result, sentence\n",
        "\n",
        "    # the predicted ID is fed back into the model\n",
        "    dec_input = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "  return result, sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "sl9zUHzg3jGI"
      },
      "outputs": [],
      "source": [
        "def translate(sentence):\n",
        "  result, sentence = evaluate(sentence)\n",
        "\n",
        "  print('Input: %s' % (sentence))\n",
        "  print('Predicted translation: {}'.format(result))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "n250XbnjOaqP"
      },
      "source": [
        "## Restore the latest checkpoint and test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "UJpT9D5_OgP6",
        "outputId": "d1628fc9-1681-48c1-840e-61677a7e49c3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f282acbe5f8>"
            ]
          },
          "execution_count": 29,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# restoring the latest checkpoint in checkpoint_dir\n",
        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "WrAM0FDomq3E",
        "outputId": "5104049f-b169-42dd-a26d-63eedcceda2d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "needed (1, 16, 7, 128)\n"
          ]
        },
        {
          "ename": "ValueError",
          "evalue": "Exception encountered when calling layer 'encoder' (type Encoder).\n\nInput 0 of layer \"gru\" is incompatible with the layer: expected ndim=3, found ndim=4. Full shape received: (1, 16, 7, 128)\n\nCall arguments received by layer 'encoder' (type Encoder):\n  • x=tf.Tensor(shape=(1, 16, 7), dtype=int32)\n  • hidden=['tf.Tensor(shape=(1, 124), dtype=float32)']",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "Cell \u001b[1;32mIn[21], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mtranslate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mu\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhace mucho frio aqui.\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
            "Cell \u001b[1;32mIn[20], line 2\u001b[0m, in \u001b[0;36mtranslate\u001b[1;34m(sentence)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtranslate\u001b[39m(sentence):\n\u001b[1;32m----> 2\u001b[0m   result, sentence \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43msentence\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      4\u001b[0m   \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mInput: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (sentence))\n\u001b[0;32m      5\u001b[0m   \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPredicted translation: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(result))\n",
            "Cell \u001b[1;32mIn[19], line 15\u001b[0m, in \u001b[0;36mevaluate\u001b[1;34m(sentence)\u001b[0m\n\u001b[0;32m     12\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m     14\u001b[0m hidden \u001b[38;5;241m=\u001b[39m [tf\u001b[38;5;241m.\u001b[39mzeros((\u001b[38;5;241m1\u001b[39m, units))]\n\u001b[1;32m---> 15\u001b[0m enc_out, enc_hidden \u001b[38;5;241m=\u001b[39m \u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     17\u001b[0m dec_hidden \u001b[38;5;241m=\u001b[39m enc_hidden\n\u001b[0;32m     18\u001b[0m dec_input \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mexpand_dims([targ_lang\u001b[38;5;241m.\u001b[39mword_index[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m<start>\u001b[39m\u001b[38;5;124m'\u001b[39m]], \u001b[38;5;241m0\u001b[39m)\n",
            "File \u001b[1;32md:\\test_locally_notebooks\\nmt\\venv\\lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[0;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[1;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
            "Cell \u001b[1;32mIn[8], line 19\u001b[0m, in \u001b[0;36mEncoder.call\u001b[1;34m(self, x, hidden)\u001b[0m\n\u001b[0;32m     17\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding(x)\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mneeded \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mx\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m---> 19\u001b[0m output, state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgru\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitial_state\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mhidden\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m output, state\n",
            "\u001b[1;31mValueError\u001b[0m: Exception encountered when calling layer 'encoder' (type Encoder).\n\nInput 0 of layer \"gru\" is incompatible with the layer: expected ndim=3, found ndim=4. Full shape received: (1, 16, 7, 128)\n\nCall arguments received by layer 'encoder' (type Encoder):\n  • x=tf.Tensor(shape=(1, 16, 7), dtype=int32)\n  • hidden=['tf.Tensor(shape=(1, 124), dtype=float32)']"
          ]
        }
      ],
      "source": [
        "translate(u'hace mucho frio aqui.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "zSx2iM36EZQZ",
        "outputId": "5b8dc076-acb5-4d62-d66f-13f8d0234adb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Input: <start> esta es mi vida . <end>\n",
            "Predicted translation: this is my life . <end> \n"
          ]
        }
      ],
      "source": [
        "translate(u'esta es mi vida.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "A3LLCx3ZE0Ls",
        "outputId": "8c970bcf-97d8-480d-ed79-cfc80c757939"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Input: <start> ¿ todavia estan en casa ? <end>\n",
            "Predicted translation: are you home ? <end> \n"
          ]
        }
      ],
      "source": [
        "translate(u'¿todavia estan en casa?')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "colab_type": "code",
        "id": "DUQVLVqUE1YW",
        "outputId": "64f3c639-9bd1-41dc-de13-b7cd7f0e3c03"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Input: <start> trata de averiguarlo . <end>\n",
            "Predicted translation: try to find out . <end> \n"
          ]
        }
      ],
      "source": [
        "# wrong translation\n",
        "translate(u'trata de averiguarlo.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "92aXUSuvwzOt"
      },
      "outputs": [],
      "source": [
        "def translate_batch(test_dataset):\n",
        "  with open('output_text.txt', 'w') as f:\n",
        "    for (inputs, targets) in test_dataset:\n",
        "      outputs = np.zeros((BATCH_SIZE, max_length_output),dtype=np.int16)\n",
        "      hidden_state = tf.zeros((BATCH_SIZE, units))\n",
        "      enc_output, dec_h = encoder(inputs, hidden_state)\n",
        "      dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "      for t in range(max_length_output):\n",
        "        preds, dec_h, _ = decoder(dec_input, dec_h, enc_output)\n",
        "        predicted_id = tf.argmax(preds, axis=1).numpy()\n",
        "        outputs[:, t] = predicted_id\n",
        "        dec_input = tf.expand_dims(predicted_id, 1)\n",
        "      outputs = targ_lang.sequences_to_texts(outputs)\n",
        "      for t, item in enumerate(outputs):\n",
        "        try:\n",
        "          i = item.index('<end>')\n",
        "          f.write(\"%s\\n\" %item[:i])\n",
        "        except: \n",
        "          f.write(\"%s \\n\" % item) # For those translated sequences which didn't correctly translated and have <end> token.\n",
        "\n",
        "outputs = translate_batch(val_dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "colab_type": "code",
        "id": "rbHSSD1dT1D0",
        "outputId": "9876c66c-90e1-43f4-ba4f-8f9bebddf894"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "walk slowly . \n",
            "i ll keep my fate . \n",
            "welcome back . \n",
            "i love to travel . \n",
            "don t touch me ! \n",
            "hold the rope . \n",
            "fantastic ! \n",
            "tom betrayed you . \n",
            "get it up . \n",
            "i was very nervous . \n",
            "5952 output_text.txt\n"
          ]
        }
      ],
      "source": [
        "!head output_text.txt\n",
        "! wc -l output_text.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "colab_type": "code",
        "id": "t2Ci8zsFbL8q",
        "outputId": "a90041ce-b3fa-449c-a410-1ccc08f659ff"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'numpy.ndarray'>\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['<start> walk more slowly . <end> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> i ll have my revenge . <end> <OOV> <OOV> <OOV>',\n",
              " '<start> welcome back . <end> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> i love reading . <end> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> don t push me ! <end> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> he let go of the rope . <end> <OOV> <OOV>',\n",
              " '<start> run ! <end> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> tom betrayed you . <end> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> hold on . <end> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV>',\n",
              " '<start> he was very nervous . <end> <OOV> <OOV> <OOV> <OOV>']"
            ]
          },
          "execution_count": 72,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "val_targets = list(val_dataset.take(1))\n",
        "val_targets = np.asarray(val_targets[0][1])\n",
        "print(type(val_targets))\n",
        "targ_lang.sequences_to_texts(val_targets)[:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "kk7sbSx9c1a9"
      },
      "source": [
        "We can see that the model worked well. Despite being not very accurate the translations, however do make some sense. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "okmWWHmMc8T6"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "4_enc_dec_with_BahdanauAttention.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}