Natural_Language_Processing_with_RNNs_and_Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Natural Language Processing with RNNs and Attention.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.2"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c5a60f634ed9421d9f525b72dbaf76bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dd0e81f1ab464c319fe22c0206af542f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_88dd596ca50f458fb66175538f81baae",
              "IPY_MODEL_1aa2bd364a5b47a1b4193016e8ce885f"
            ]
          }
        },
        "dd0e81f1ab464c319fe22c0206af542f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88dd596ca50f458fb66175538f81baae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_17f240bc009c4b11b98ba6ea7beae62d",
            "_dom_classes": [],
            "description": "Dl Completed...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22954974233b43d8b1215248690b71c5"
          }
        },
        "1aa2bd364a5b47a1b4193016e8ce885f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2b5a2087ab1d45adbe76407063941f98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:08&lt;00:00,  8.95s/ url]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e7b937d8f4e43bdbb1806b748028930"
          }
        },
        "17f240bc009c4b11b98ba6ea7beae62d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22954974233b43d8b1215248690b71c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b5a2087ab1d45adbe76407063941f98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e7b937d8f4e43bdbb1806b748028930": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b30ea83194ef453691296fcf9bd8ec07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f1e45d01facd4057a8912f30fd3452d7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f7d097680edb4562820b6ffc4bf7e255",
              "IPY_MODEL_496f1f52f9cd4f22a4e43c4542593ca1"
            ]
          }
        },
        "f1e45d01facd4057a8912f30fd3452d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7d097680edb4562820b6ffc4bf7e255": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0b50495ab0f44ff7be981074eba6e558",
            "_dom_classes": [],
            "description": "Dl Size...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_12efa455619e455fbeff88c4ec672661"
          }
        },
        "496f1f52f9cd4f22a4e43c4542593ca1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c3c4050e4d6a443781bf1ffa7842fdb5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 80/80 [00:08&lt;00:00,  8.98 MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88c7149cebe6482c920448891aba4499"
          }
        },
        "0b50495ab0f44ff7be981074eba6e558": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "12efa455619e455fbeff88c4ec672661": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c3c4050e4d6a443781bf1ffa7842fdb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88c7149cebe6482c920448891aba4499": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a446cdf11d9d4e228ba74f3923733dcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_436e974301ee47d38ef691dad9879b5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1a1bc88682014205ac6913b4b524be13",
              "IPY_MODEL_e1d23fd7f44349cab54d51748033b003"
            ]
          }
        },
        "436e974301ee47d38ef691dad9879b5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a1bc88682014205ac6913b4b524be13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a4187d00a38f47beb94d1c8bf5945587",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a6af399a87da461aa098254d2e25b0ba"
          }
        },
        "e1d23fd7f44349cab54d51748033b003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ec7725580cf34e33ad89737502c162a1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:17&lt;00:00, 2401.89 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d869958846a4957b19694c7339de68c"
          }
        },
        "a4187d00a38f47beb94d1c8bf5945587": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a6af399a87da461aa098254d2e25b0ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec7725580cf34e33ad89737502c162a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d869958846a4957b19694c7339de68c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9aecbee6667a4b7f96de3ef0f5cbac15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c5b1d5b44df1443989a5063a9a8f57fd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2c89c41034b7465fbd6095be6f61ff01",
              "IPY_MODEL_659c2029a78341748509d99aba6628e6"
            ]
          }
        },
        "c5b1d5b44df1443989a5063a9a8f57fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c89c41034b7465fbd6095be6f61ff01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_267debd1be0d49ad92596cb6d4af548d",
            "_dom_classes": [],
            "description": " 40%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 25000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10113,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31a71a1ac92c4c77a294f3f902e894ec"
          }
        },
        "659c2029a78341748509d99aba6628e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c60a48ce3b564cb8aa16729923ffdb5b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10113/25000 [00:00&lt;00:00, 101127.93 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1490442212034bad97f6f775c516b203"
          }
        },
        "267debd1be0d49ad92596cb6d4af548d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31a71a1ac92c4c77a294f3f902e894ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c60a48ce3b564cb8aa16729923ffdb5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1490442212034bad97f6f775c516b203": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48ed7fa60135403ab22797f7456e801c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fca5fd5ad54d4275b9ff1d10eff82668",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bf594ddc253043bd9b5e2661ec9da7ee",
              "IPY_MODEL_a18dfe4e5393494d900d9c8b8144e09f"
            ]
          }
        },
        "fca5fd5ad54d4275b9ff1d10eff82668": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bf594ddc253043bd9b5e2661ec9da7ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_294e2e546e654499bffd23eabf8eb54c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cd7543f159b40eb88c36ba680add915"
          }
        },
        "a18dfe4e5393494d900d9c8b8144e09f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a51c99bf26aa46179a2b0844044c62b1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/0 [00:17&lt;00:00, 2416.89 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42cf5289ad0d4b28bc9d76548c2a9712"
          }
        },
        "294e2e546e654499bffd23eabf8eb54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cd7543f159b40eb88c36ba680add915": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a51c99bf26aa46179a2b0844044c62b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42cf5289ad0d4b28bc9d76548c2a9712": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3682132f5ea14c0885f224324793b108": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d5af7e19bff5415aaf1e8bb1e6216a40",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_86223ffe45094f919e51f39554f54bc0",
              "IPY_MODEL_f6193d463a9543b7a25f79d57b4231ed"
            ]
          }
        },
        "d5af7e19bff5415aaf1e8bb1e6216a40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86223ffe45094f919e51f39554f54bc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85472e539a664dfcb523e1add0854086",
            "_dom_classes": [],
            "description": " 40%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 25000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 9996,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e62bfeb5c0fa4e00929c75b84ce50609"
          }
        },
        "f6193d463a9543b7a25f79d57b4231ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c5bbb825c0094ff98329278bbd4ec229",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9996/25000 [00:00&lt;00:00, 99959.14 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_71b48f981ba141739bd84c623e85a400"
          }
        },
        "85472e539a664dfcb523e1add0854086": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e62bfeb5c0fa4e00929c75b84ce50609": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5bbb825c0094ff98329278bbd4ec229": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "71b48f981ba141739bd84c623e85a400": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58781d9b89cb416fbdf931f055cd3669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a45aaeef5d9447f4baeff566328a8a16",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7746f8b3bbd544c58bf2e7bec063ea54",
              "IPY_MODEL_3d55f10eea9a4474b154236b1f086fb4"
            ]
          }
        },
        "a45aaeef5d9447f4baeff566328a8a16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7746f8b3bbd544c58bf2e7bec063ea54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bc14ce02a599415199e2f1714253a094",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_94486c28a8594785adecf1e42ad71cfc"
          }
        },
        "3d55f10eea9a4474b154236b1f086fb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2433cb5f3dcd4e439e486a795d2f0ef1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 50000/0 [00:25&lt;00:00, 2378.49 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7561ff75ee474b4eabb534ddf7bda5b0"
          }
        },
        "bc14ce02a599415199e2f1714253a094": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "94486c28a8594785adecf1e42ad71cfc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2433cb5f3dcd4e439e486a795d2f0ef1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7561ff75ee474b4eabb534ddf7bda5b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f85060156194c098868968103dd7df2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fc5d653aa07a4395a7fc4a237ae58431",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0fe3d299cf8f4005b53b4b9542471063",
              "IPY_MODEL_9dd1fa7632174187b0beabe7ec780426"
            ]
          }
        },
        "fc5d653aa07a4395a7fc4a237ae58431": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0fe3d299cf8f4005b53b4b9542471063": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7651211a07aa460fa8e854b66527feb0",
            "_dom_classes": [],
            "description": " 94%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 50000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 47136,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d8353f8c93644809e4bbe3cdac8021c"
          }
        },
        "9dd1fa7632174187b0beabe7ec780426": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_32012cf6be91420882de7bf1ea4955f7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 47136/50000 [00:00&lt;00:00, 83369.95 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_151e70c8af4d4206b1f2b693aecc8f6a"
          }
        },
        "7651211a07aa460fa8e854b66527feb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d8353f8c93644809e4bbe3cdac8021c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "32012cf6be91420882de7bf1ea4955f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "151e70c8af4d4206b1f2b693aecc8f6a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shrey-Viradiya/HandsOnMachineLearning/blob/master/Natural_Language_Processing_with_RNNs_and_Attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ESPg5c8Cx96S",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "outputId": "8019326d-18f8-4e6c-9daa-903d020a0500"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jun 29 17:03:41 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.36.06    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   70C    P8    36W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "stABES0AiSXH"
      },
      "source": [
        "# Natural Language Processing with RNNs and Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EuRKe4pyiQ_B"
      },
      "source": [
        "## Generating Shakespearean Text Using a Character RNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4vQ61owiiVR4"
      },
      "source": [
        "### Creating the Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Gl7dBA4kiiKe",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FRkJU53B9o8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
        "for gpu in gpus:\n",
        "        tf.config.experimental.set_memory_growth(gpu, True)\n",
        "tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024)])"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8t2cARv7iU9r",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "dc67c3c2-adb1-4ddd-a765-eef5f67fb2aa"
      },
      "source": [
        "shakespeare_url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n",
        "filepath = keras.utils.get_file(\"shakespeare.txt\", shakespeare_url)\n",
        "with open(filepath) as f:\n",
        "    shakespeare_text = f.read()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\n",
            "1122304/1115394 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ElmNVAvFp1vU",
        "colab": {}
      },
      "source": [
        "tokenizer = keras.preprocessing.text.Tokenizer(char_level=True)\n",
        "tokenizer.fit_on_texts(shakespeare_text)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xkte17tuqI0n",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "21ec6c3c-86f1-4f85-ed00-9c4685c963cf"
      },
      "source": [
        "tokenizer.texts_to_sequences(['First'])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[20, 6, 9, 8, 3]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LpjigMYlq-RX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0676285e-dd47-411f-a52e-afca958782c7"
      },
      "source": [
        "tokenizer.sequences_to_texts([[20,6,9,8,3]])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['f i r s t']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bVxVSQ-CrCGu",
        "colab": {}
      },
      "source": [
        "max_id = len(tokenizer.word_index)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TeagwkiXrHSj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ec127fe8-ccdd-4d26-ebf1-8717dbcd2ba6"
      },
      "source": [
        "max_id"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "39"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8ryozElBrIIs",
        "colab": {}
      },
      "source": [
        "dataset_size = tokenizer.document_count"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "5_bpeXf2rKDt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8a0dfc03-5336-40dc-fdfc-a03f39edf4ef"
      },
      "source": [
        "dataset_size"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1115394"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NOQdmB-erK1R",
        "colab": {}
      },
      "source": [
        "[encoded] = np.array(tokenizer.texts_to_sequences([shakespeare_text])) - 1"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "D5gyOOsurZ2b"
      },
      "source": [
        "### How to Split a Sequential Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "R11NyOFSsQRs"
      },
      "source": [
        "Let’s take the first 90% of the text for the training set (keeping the rest for the validation set and the test set), and create a tf.data.Dataset that will return each character one by one from this set:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9VrkViw_rVjZ",
        "colab": {}
      },
      "source": [
        "train_size = dataset_size * 90 // 100\n",
        "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iOGAHX5_sxu-"
      },
      "source": [
        "### Chopping the Sequential Dataset into Multiple Windows"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "PcR3LfbWs3O_"
      },
      "source": [
        "The training set now consists of a single sequence of over a million characters, so we can’t just train the neural network directly on it: the RNN would be equivalent to a deep net with over a million layers, and we would have a single (very long) instance to train it. Instead, we will use the dataset’s window() method to convert this long sequence of characters into many smaller windows of text. Every instance in the dataset will be a fairly short substring of the whole text, and the RNN will be unrolled only over the length of these substrings. This is called truncated backpropagation through time. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Wszb3DllsxhE",
        "colab": {}
      },
      "source": [
        "n_steps = 100\n",
        "window_length = n_steps + 1"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pBI9ZLJOsVlQ",
        "colab": {}
      },
      "source": [
        "dataset = dataset.window(window_length, shift=1, drop_remainder=True)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Kc6ppRPCtlqX"
      },
      "source": [
        "The window() method creates a dataset that contains windows, each of which is also represented as a dataset. It’s a nested dataset, analogous to a list of lists. This is useful when you want to transform each window by calling its dataset methods (e.g., to shuffle them or batch them). However, we cannot use a nested dataset directly for training, as our model will expect tensors as input, not datasets. So, we must call the flat_map() method: it converts a nested dataset into a flat dataset (one that does not contain datasets)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LTBpzOzgtLzI",
        "colab": {}
      },
      "source": [
        "dataset = dataset.flat_map(lambda window: window.batch(window_length))"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MJqX1OKiu1oL"
      },
      "source": [
        "We need to shuffle these windows. Then we can batch the windows and separate the inputs (the first 100 characters) from the target (the last character):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ojTp65F4tyFE",
        "colab": {}
      },
      "source": [
        "batch_size = 32\n",
        "dataset = dataset.shuffle(10000).batch(batch_size)\n",
        "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "3DFyRcMOwGgm"
      },
      "source": [
        "Categorical input features should generally be encoded, usually as one-hot vectors or as embeddings. Here, we will encode each character using a one-hot vector because there are fairly few distinct characters (only 39):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gxdbhgS6u2yE",
        "colab": {}
      },
      "source": [
        "dataset = dataset.map(\n",
        "    lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "w5a8pbojwKHc",
        "colab": {}
      },
      "source": [
        "dataset = dataset.prefetch(1)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "FnusNQaPwPfr"
      },
      "source": [
        "### Building and Training the Char-RNN Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zDHNvVE6zFaO",
        "colab": {}
      },
      "source": [
        "import os\n",
        "\n",
        "checkpoint_dir = './training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n",
        "\n",
        "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_prefix,\n",
        "    save_weights_only=True)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1brVMPhFB9rM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NvidiaUtilizationCallback(keras.callbacks.Callback):\n",
        "    def on_epoch_begin(self, epoch, logs):\n",
        "        text = !nvidia-smi\n",
        "        text = text[9][60:65] + ' GPU utilization'\n",
        "        print(text)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fwIgGDCswNrs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "78abd3cd-1a3f-4e11-a269-6df9c8c7cc1b"
      },
      "source": [
        "model = keras.models.Sequential([\n",
        "    keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id]),\n",
        "    keras.layers.TimeDistributed(keras.layers.Dense(max_id, activation='softmax'))    \n",
        "])\n",
        "\n",
        "model.compile(loss = 'sparse_categorical_crossentropy', optimizer = 'adam')\n",
        "steps_per_epoch = train_size // batch_size // n_steps\n",
        "\n",
        "history = model.fit(dataset, epochs = 40, steps_per_epoch=steps_per_epoch, callbacks=[checkpoint_callback, NvidiaUtilizationCallback()])"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%  GPU utilization\n",
            "Epoch 1/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 2.5693\n",
            "  0%  GPU utilization\n",
            "Epoch 2/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 2.0548\n",
            "  0%  GPU utilization\n",
            "Epoch 3/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.8476\n",
            "  0%  GPU utilization\n",
            "Epoch 4/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.6848\n",
            "  0%  GPU utilization\n",
            "Epoch 5/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.5260\n",
            "  0%  GPU utilization\n",
            "Epoch 6/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3821\n",
            "  0%  GPU utilization\n",
            "Epoch 7/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3227\n",
            "  0%  GPU utilization\n",
            "Epoch 8/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.2815\n",
            "  0%  GPU utilization\n",
            "Epoch 9/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.2668\n",
            "  0%  GPU utilization\n",
            "Epoch 10/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.2625\n",
            "  0%  GPU utilization\n",
            "Epoch 11/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2536\n",
            "  0%  GPU utilization\n",
            "Epoch 12/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.2652\n",
            "  0%  GPU utilization\n",
            "Epoch 13/40\n",
            "313/313 [==============================] - 8s 24ms/step - loss: 1.2787\n",
            "  0%  GPU utilization\n",
            "Epoch 14/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3122\n",
            "  0%  GPU utilization\n",
            "Epoch 15/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3424\n",
            "  0%  GPU utilization\n",
            "Epoch 16/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3514\n",
            "  0%  GPU utilization\n",
            "Epoch 17/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.3148\n",
            "  0%  GPU utilization\n",
            "Epoch 18/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2892\n",
            "  0%  GPU utilization\n",
            "Epoch 19/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2885\n",
            "  0%  GPU utilization\n",
            "Epoch 20/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2427\n",
            "  0%  GPU utilization\n",
            "Epoch 21/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2620\n",
            "  0%  GPU utilization\n",
            "Epoch 22/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.2292\n",
            "  0%  GPU utilization\n",
            "Epoch 23/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.1875\n",
            "  0%  GPU utilization\n",
            "Epoch 24/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.1932\n",
            "  0%  GPU utilization\n",
            "Epoch 25/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2366\n",
            "  0%  GPU utilization\n",
            "Epoch 26/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2361\n",
            "  0%  GPU utilization\n",
            "Epoch 27/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2480\n",
            "  0%  GPU utilization\n",
            "Epoch 28/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.1972\n",
            "  0%  GPU utilization\n",
            "Epoch 29/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.1622\n",
            "  0%  GPU utilization\n",
            "Epoch 30/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2335\n",
            "  0%  GPU utilization\n",
            "Epoch 31/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2281\n",
            "  0%  GPU utilization\n",
            "Epoch 32/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.2707\n",
            "  0%  GPU utilization\n",
            "Epoch 33/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.2597\n",
            "  0%  GPU utilization\n",
            "Epoch 34/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2736\n",
            "  0%  GPU utilization\n",
            "Epoch 35/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2723\n",
            "  0%  GPU utilization\n",
            "Epoch 36/40\n",
            "313/313 [==============================] - 7s 24ms/step - loss: 1.2659\n",
            "  0%  GPU utilization\n",
            "Epoch 37/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.2302\n",
            "  0%  GPU utilization\n",
            "Epoch 38/40\n",
            "313/313 [==============================] - 7s 23ms/step - loss: 1.2558\n",
            "  0%  GPU utilization\n",
            "Epoch 39/40\n",
            "313/313 [==============================] - 7s 22ms/step - loss: 1.2324\n",
            "  0%  GPU utilization\n",
            "Epoch 40/40\n",
            "313/313 [==============================] - 7s 21ms/step - loss: 1.2458\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "50SESxWMr6fU"
      },
      "source": [
        "### Using the Model to Generate Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "f31S40l_FAYM",
        "colab": {}
      },
      "source": [
        "def preprocess(texts):\n",
        "    X = np.array(tokenizer.texts_to_sequences(texts)) - 1\n",
        "    return tf.one_hot(X, max_id)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9RL2ADK0r_Pi",
        "colab": {}
      },
      "source": [
        "X_new = preprocess([\"How are yo\"])"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JBIEqfpSsAuk",
        "colab": {}
      },
      "source": [
        "Y_pred = np.argmax(model.predict(X_new), axis=-1)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0FkLx6JSsCV7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1c8ad689-428e-4d3a-9db0-022cd728dc86"
      },
      "source": [
        "tokenizer.sequences_to_texts(Y_pred + 1)[0][-1]"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic": {
              "type": "string"
            },
            "text/plain": [
              "'u'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "SXfDGjVesYau"
      },
      "source": [
        "### Generating Fake Shakespearean Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "I5d09rGqtnP3",
        "colab": {}
      },
      "source": [
        "def next_char(text, temperature=1):\n",
        "    X_new = preprocess([text])\n",
        "    y_proba = model.predict(X_new)[0, -1:, :]\n",
        "    rescaled_logits = tf.math.log(y_proba) / temperature\n",
        "    char_id = tf.random.categorical(rescaled_logits, num_samples=1) + 1\n",
        "    return tokenizer.sequences_to_texts(char_id.numpy())[0]"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0FHDnAKQtnih",
        "colab": {}
      },
      "source": [
        "def complete_text(text, n_chars=50, temperature=1):\n",
        "    for _ in range(n_chars):\n",
        "        text += next_char(text, temperature)\n",
        "    return text"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OSYXR11htu2p",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 836
        },
        "outputId": "0a41e037-eef4-4adb-99b1-137cea850df1"
      },
      "source": [
        "text_1 = complete_text(\"t\", temperature=0.2)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 7 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 8 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 10 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 12 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sk_HXTHytxfa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b5a8fb21-5d7b-4fd9-9e4d-d33bf1fc583b"
      },
      "source": [
        "print(text_1)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "the throw at all this present, and not this not thi\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7ev6Xtcpt67_",
        "colab": {}
      },
      "source": [
        "text_2 = complete_text(\"w\", temperature=1)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YvHwz4d1uGnL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "ab40ea67-0300-485f-8aaf-a3fba9ac68dc"
      },
      "source": [
        "print(text_2)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "wall is must his rubose of bolingbroke\n",
            "of say the l\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "B_wC1AgauLWY",
        "colab": {}
      },
      "source": [
        "text_3 = complete_text(\"i\", temperature=2)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YfOisbHwuY2L",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "6247e031-9ea0-4829-adba-220aba40f60d"
      },
      "source": [
        "print(text_3)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "id-sennd.\n",
            "if whe not! that pissy, thinksperes\n",
            "con,a\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-3bDIrYAuanR",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a44ac5b5-3217-4325-d6a3-3b1ca3f557cd"
      },
      "source": [
        "text_4 = complete_text(\"I shall love\", temperature=0.5, n_chars=100)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 144 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 145 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 146 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 147 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 148 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 149 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 150 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "FKj-sX03upqp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "a0d2179d-885a-4ea5-a24b-46c7dc9a9274"
      },
      "source": [
        "print(text_4)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I shall love and tell all thou werk head?\n",
            "\n",
            "duke of york:\n",
            "seer would confood,\n",
            "and lies,\n",
            "that king of my trialter \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "frbFoaPsuy8R",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "7b8cf59b-5263-4774-9b12-15082c0eb26c"
      },
      "source": [
        "text_5 = complete_text(\"love\", temperature=1, n_chars = 150)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 113 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 114 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 115 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 116 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 117 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 118 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 119 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f9431512b70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "v3IqE8A2vKnu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "12b48e53-b881-43c8-f4ae-f2fe5d775c03"
      },
      "source": [
        "print(text_5)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "love,\n",
            "or glard, mind;\n",
            "or your in the liah handy;\n",
            "what was comfort citilip, in joan our heads? who ere tenderen war dithersed to his noght may he secin; an\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "IB0BM5rXwCHq"
      },
      "source": [
        "### Stateful RNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QPoDWfb1wCmT"
      },
      "source": [
        "First, note that a stateful RNN only makes sense if each input sequence in a batch starts exactly where the corresponding sequence in the previous batch left off. So the first thing we need to do to build a stateful RNN is to use sequential and nonoverlapping input sequences (rather than the shuffled and overlapping sequences we used to train stateless RNNs). When creating the Dataset, we must therefore use shift=n_steps (instead of shift=1) when calling the window() method. Moreover, we must obviously not call the shuffle() method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_U1thQsdwsNT",
        "colab": {}
      },
      "source": [
        "tf.random.set_seed(42)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WSHiXNCovMJD",
        "colab": {}
      },
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices(encoded[:train_size])\n",
        "dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n",
        "dataset = dataset.flat_map(lambda window: window.batch(window_length))\n",
        "dataset = dataset.batch(1)\n",
        "dataset = dataset.map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n",
        "dataset = dataset.map(\n",
        "    lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n",
        "dataset = dataset.prefetch(1)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QymjkFnpz0VD",
        "colab": {}
      },
      "source": [
        "batch_size = 32\n",
        "encoded_parts = np.array_split(encoded[:train_size], batch_size)\n",
        "datasets = []\n",
        "for encoded_part in encoded_parts:\n",
        "    dataset = tf.data.Dataset.from_tensor_slices(encoded_part)\n",
        "    dataset = dataset.window(window_length, shift=n_steps, drop_remainder=True)\n",
        "    dataset = dataset.flat_map(lambda window: window.batch(window_length))\n",
        "    datasets.append(dataset)\n",
        "dataset = tf.data.Dataset.zip(tuple(datasets)).map(lambda *windows: tf.stack(windows))\n",
        "dataset = dataset.repeat().map(lambda windows: (windows[:, :-1], windows[:, 1:]))\n",
        "dataset = dataset.map(\n",
        "    lambda X_batch, Y_batch: (tf.one_hot(X_batch, depth=max_id), Y_batch))\n",
        "dataset = dataset.prefetch(1)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "daxlamGvwIXp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "bf4a11ff-3743-43cb-dd6c-ad62a822fcd3"
      },
      "source": [
        "model = keras.models.Sequential([\n",
        "    keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
        "                     dropout=0.2, recurrent_dropout=0.2,\n",
        "                     batch_input_shape=[batch_size, None, max_id]),\n",
        "    keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
        "                     dropout=0.2, recurrent_dropout=0.2),\n",
        "    keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n",
        "                                                    activation=\"softmax\"))\n",
        "])\n",
        "\n",
        "class ResetStatesCallback(keras.callbacks.Callback):\n",
        "    def on_epoch_begin(self, epoch, logs):\n",
        "        self.model.reset_states()"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Layer gru_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
            "WARNING:tensorflow:Layer gru_2 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Kcs_lXKhzkk4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "1e08df3e-0504-4a67-968e-4302fc802454"
      },
      "source": [
        "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n",
        "steps_per_epoch = train_size // batch_size // n_steps\n",
        "history = model.fit(dataset, steps_per_epoch=steps_per_epoch, epochs=40,\n",
        "                    callbacks=[ResetStatesCallback(), NvidiaUtilizationCallback()])"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%  GPU utilization\n",
            "Epoch 1/40\n",
            "313/313 [==============================] - 103s 329ms/step - loss: 2.6128\n",
            "  0%  GPU utilization\n",
            "Epoch 2/40\n",
            "313/313 [==============================] - 102s 327ms/step - loss: 2.1758\n",
            "  0%  GPU utilization\n",
            "Epoch 3/40\n",
            "313/313 [==============================] - 103s 330ms/step - loss: 2.6733\n",
            "  0%  GPU utilization\n",
            "Epoch 4/40\n",
            "313/313 [==============================] - 103s 328ms/step - loss: 2.4080\n",
            "  0%  GPU utilization\n",
            "Epoch 5/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 2.2096\n",
            "  0%  GPU utilization\n",
            "Epoch 6/40\n",
            "313/313 [==============================] - 103s 329ms/step - loss: 2.1379\n",
            "  0%  GPU utilization\n",
            "Epoch 7/40\n",
            "313/313 [==============================] - 101s 324ms/step - loss: 2.0510\n",
            "  0%  GPU utilization\n",
            "Epoch 8/40\n",
            "313/313 [==============================] - 102s 325ms/step - loss: 2.0023\n",
            "  0%  GPU utilization\n",
            "Epoch 9/40\n",
            "313/313 [==============================] - 102s 327ms/step - loss: 1.9752\n",
            "  0%  GPU utilization\n",
            "Epoch 10/40\n",
            "313/313 [==============================] - 101s 324ms/step - loss: 1.9436\n",
            "  0%  GPU utilization\n",
            "Epoch 11/40\n",
            "313/313 [==============================] - 102s 325ms/step - loss: 1.9053\n",
            "  0%  GPU utilization\n",
            "Epoch 12/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 1.8778\n",
            "  0%  GPU utilization\n",
            "Epoch 13/40\n",
            "313/313 [==============================] - 101s 324ms/step - loss: 1.8531\n",
            "  0%  GPU utilization\n",
            "Epoch 14/40\n",
            "313/313 [==============================] - 101s 324ms/step - loss: 1.8330\n",
            "  0%  GPU utilization\n",
            "Epoch 15/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 1.8145\n",
            "  0%  GPU utilization\n",
            "Epoch 16/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.7982\n",
            "  0%  GPU utilization\n",
            "Epoch 17/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.7844\n",
            "  0%  GPU utilization\n",
            "Epoch 18/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 1.7736\n",
            "  0%  GPU utilization\n",
            "Epoch 19/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.7618\n",
            "  0%  GPU utilization\n",
            "Epoch 20/40\n",
            "313/313 [==============================] - 101s 322ms/step - loss: 1.7511\n",
            "  0%  GPU utilization\n",
            "Epoch 21/40\n",
            "313/313 [==============================] - 102s 327ms/step - loss: 1.7425\n",
            "  0%  GPU utilization\n",
            "Epoch 22/40\n",
            "313/313 [==============================] - 102s 325ms/step - loss: 1.7324\n",
            "  0%  GPU utilization\n",
            "Epoch 23/40\n",
            "313/313 [==============================] - 103s 328ms/step - loss: 1.7250\n",
            "  0%  GPU utilization\n",
            "Epoch 24/40\n",
            "313/313 [==============================] - 103s 328ms/step - loss: 1.7157\n",
            "  0%  GPU utilization\n",
            "Epoch 25/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 1.7100\n",
            "  0%  GPU utilization\n",
            "Epoch 26/40\n",
            "313/313 [==============================] - 102s 325ms/step - loss: 1.7016\n",
            "  0%  GPU utilization\n",
            "Epoch 27/40\n",
            "313/313 [==============================] - 103s 329ms/step - loss: 1.6947\n",
            "  0%  GPU utilization\n",
            "Epoch 28/40\n",
            "313/313 [==============================] - 102s 326ms/step - loss: 1.6877\n",
            "  0%  GPU utilization\n",
            "Epoch 29/40\n",
            "313/313 [==============================] - 102s 325ms/step - loss: 1.6821\n",
            "  0%  GPU utilization\n",
            "Epoch 30/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.6764\n",
            "  0%  GPU utilization\n",
            "Epoch 31/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.6694\n",
            "  0%  GPU utilization\n",
            "Epoch 32/40\n",
            "313/313 [==============================] - 101s 324ms/step - loss: 1.6653\n",
            "  0%  GPU utilization\n",
            "Epoch 33/40\n",
            "313/313 [==============================] - 100s 320ms/step - loss: 1.6601\n",
            "  0%  GPU utilization\n",
            "Epoch 34/40\n",
            "313/313 [==============================] - 100s 320ms/step - loss: 1.6544\n",
            "  0%  GPU utilization\n",
            "Epoch 35/40\n",
            "313/313 [==============================] - 101s 323ms/step - loss: 1.6490\n",
            "  0%  GPU utilization\n",
            "Epoch 36/40\n",
            "313/313 [==============================] - 100s 320ms/step - loss: 1.6445\n",
            "  0%  GPU utilization\n",
            "Epoch 37/40\n",
            "313/313 [==============================] - 100s 318ms/step - loss: 1.6404\n",
            "  0%  GPU utilization\n",
            "Epoch 38/40\n",
            "313/313 [==============================] - 100s 319ms/step - loss: 1.6362\n",
            "  0%  GPU utilization\n",
            "Epoch 39/40\n",
            "313/313 [==============================] - 99s 316ms/step - loss: 1.6311\n",
            "  0%  GPU utilization\n",
            "Epoch 40/40\n",
            "313/313 [==============================] - 99s 316ms/step - loss: 1.6269\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "cCR5RTa12cHl"
      },
      "source": [
        "\n",
        "To use the model with different batch sizes, we need to create a stateless copy. We can get rid of dropout since it is only used during training:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vLDEEHcb2bxa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "4b5eae33-d3d3-42a4-916b-69268967491c"
      },
      "source": [
        "stateless_model = keras.models.Sequential([\n",
        "    keras.layers.GRU(128, return_sequences=True, input_shape=[None, max_id], dropout=0.2, recurrent_dropout=0.2,),\n",
        "    keras.layers.GRU(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2,),\n",
        "    keras.layers.TimeDistributed(keras.layers.Dense(max_id, activation=\"softmax\"))\n",
        "])"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Layer gru_3 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
            "WARNING:tensorflow:Layer gru_4 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "DQanuXQQ2g6M"
      },
      "source": [
        "\n",
        "To set the weights, we first need to build the model (so the weights get created):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LVfV9M7dz5Gb",
        "colab": {}
      },
      "source": [
        "stateless_model.build(tf.TensorShape([None, None, max_id]))"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "EYkHzEan2jsZ",
        "colab": {}
      },
      "source": [
        "stateless_model.set_weights(model.get_weights())\n",
        "model = stateless_model"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RC28zkFqB9tH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 836
        },
        "outputId": "3fdd3aa4-b027-4363-8695-9a2e21ebf410"
      },
      "source": [
        "text_1 = complete_text(\"t\", temperature=0.2)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 7 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 8 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 10 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gVPKtUgaB9tR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "a85af9ff-545c-4485-fcc2-4d87d126644c"
      },
      "source": [
        "print(text_1)"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "there is a bear\n",
            "the provost to the duke of the coun\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_jjX0EBiB9tT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_2 = complete_text(\"w\", temperature=1)"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZjJ6lWPkB9tX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "cf9fae0f-1cf4-4e09-b6b9-4bdfc32c4de9"
      },
      "source": [
        "print(text_2)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "wick:\n",
            "i crobje constrict joy their rastalpent me\n",
            "th\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nyFnEB-6B9ta",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_3 = complete_text(\"i\", temperature=2)"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1XmUvUM0B9tf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "cd5a7d91-2250-4599-b0bc-9787d5fcc379"
      },
      "source": [
        "print(text_3)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "iv!\n",
            "well, judgice bevilg deai. ould-yaarm-'troan,\n",
            "m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EXbq9TjB9tj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3f531077-a369-4dca-ba71-a9dcfd134243"
      },
      "source": [
        "text_4 = complete_text(\"I shall love\", temperature=0.5, n_chars=100)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 144 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 145 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 146 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 147 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 148 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 149 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 150 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "860ha8uPB9ts",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "dc76441d-1fd3-49c7-f5ff-7a15d047ab52"
      },
      "source": [
        "print(text_4)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I shall love here to shall as it it.\n",
            "\n",
            "leontes:\n",
            "he therefore he heart their pastion and in the child\n",
            "to the cause\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXle3ts3B9tw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "dfd50d83-61d1-43ca-f556-79188692c06b"
      },
      "source": [
        "text_5 = complete_text(\"love\", temperature=1, n_chars = 150)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:5 out of the last 113 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:6 out of the last 114 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:7 out of the last 115 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:8 out of the last 116 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:9 out of the last 117 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:10 out of the last 118 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 119 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
            "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f93d8e78730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "su69q6wzB9t0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "2e08e2e5-4a65-4be0-c7a8-4c4ac5964433"
      },
      "source": [
        "print(text_5)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "love\n",
            "the kindled mishake him of you nevense,\n",
            "these ears me: bin our heaven, he had he\n",
            "death, by our with courthrop's pervort friends as these have now lov\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Mo4UkBVf2xU_"
      },
      "source": [
        "## Sentiment Analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "acfDYjhz2sDd",
        "colab": {}
      },
      "source": [
        "tf.random.set_seed(42)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CwuajGCU3AJX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "4f8a3217-21ed-456a-dbd4-f8dcabcf0862"
      },
      "source": [
        "(X_train, y_test), (X_valid, y_test) = keras.datasets.imdb.load_data()"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
            "17465344/17464789 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jybapjNZ3Dnh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7523e7e7-7492-46fc-f95b-1dbd86c6297c"
      },
      "source": [
        "X_train[0][:10]"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7g-tfLqp3FVN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "2a723b3c-d11b-413b-831c-71d9452de575"
      },
      "source": [
        "word_index = keras.datasets.imdb.get_word_index()\n",
        "id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n",
        "for id_, token in enumerate((\"<pad>\", \"<sos>\", \"<unk>\")):\n",
        "    id_to_word[id_] = token\n",
        "\" \".join([id_to_word[id_] for id_ in X_train[0][:10]])"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json\n",
            "1646592/1641221 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic": {
              "type": "string"
            },
            "text/plain": [
              "'<sos> this film was just brilliant casting location scenery story'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lTFYuRzN581s",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353,
          "referenced_widgets": [
            "c5a60f634ed9421d9f525b72dbaf76bc",
            "dd0e81f1ab464c319fe22c0206af542f",
            "88dd596ca50f458fb66175538f81baae",
            "1aa2bd364a5b47a1b4193016e8ce885f",
            "17f240bc009c4b11b98ba6ea7beae62d",
            "22954974233b43d8b1215248690b71c5",
            "2b5a2087ab1d45adbe76407063941f98",
            "7e7b937d8f4e43bdbb1806b748028930",
            "b30ea83194ef453691296fcf9bd8ec07",
            "f1e45d01facd4057a8912f30fd3452d7",
            "f7d097680edb4562820b6ffc4bf7e255",
            "496f1f52f9cd4f22a4e43c4542593ca1",
            "0b50495ab0f44ff7be981074eba6e558",
            "12efa455619e455fbeff88c4ec672661",
            "c3c4050e4d6a443781bf1ffa7842fdb5",
            "88c7149cebe6482c920448891aba4499",
            "a446cdf11d9d4e228ba74f3923733dcd",
            "436e974301ee47d38ef691dad9879b5d",
            "1a1bc88682014205ac6913b4b524be13",
            "e1d23fd7f44349cab54d51748033b003",
            "a4187d00a38f47beb94d1c8bf5945587",
            "a6af399a87da461aa098254d2e25b0ba",
            "ec7725580cf34e33ad89737502c162a1",
            "5d869958846a4957b19694c7339de68c",
            "9aecbee6667a4b7f96de3ef0f5cbac15",
            "c5b1d5b44df1443989a5063a9a8f57fd",
            "2c89c41034b7465fbd6095be6f61ff01",
            "659c2029a78341748509d99aba6628e6",
            "267debd1be0d49ad92596cb6d4af548d",
            "31a71a1ac92c4c77a294f3f902e894ec",
            "c60a48ce3b564cb8aa16729923ffdb5b",
            "1490442212034bad97f6f775c516b203",
            "48ed7fa60135403ab22797f7456e801c",
            "fca5fd5ad54d4275b9ff1d10eff82668",
            "bf594ddc253043bd9b5e2661ec9da7ee",
            "a18dfe4e5393494d900d9c8b8144e09f",
            "294e2e546e654499bffd23eabf8eb54c",
            "2cd7543f159b40eb88c36ba680add915",
            "a51c99bf26aa46179a2b0844044c62b1",
            "42cf5289ad0d4b28bc9d76548c2a9712",
            "3682132f5ea14c0885f224324793b108",
            "d5af7e19bff5415aaf1e8bb1e6216a40",
            "86223ffe45094f919e51f39554f54bc0",
            "f6193d463a9543b7a25f79d57b4231ed",
            "85472e539a664dfcb523e1add0854086",
            "e62bfeb5c0fa4e00929c75b84ce50609",
            "c5bbb825c0094ff98329278bbd4ec229",
            "71b48f981ba141739bd84c623e85a400",
            "58781d9b89cb416fbdf931f055cd3669",
            "a45aaeef5d9447f4baeff566328a8a16",
            "7746f8b3bbd544c58bf2e7bec063ea54",
            "3d55f10eea9a4474b154236b1f086fb4",
            "bc14ce02a599415199e2f1714253a094",
            "94486c28a8594785adecf1e42ad71cfc",
            "2433cb5f3dcd4e439e486a795d2f0ef1",
            "7561ff75ee474b4eabb534ddf7bda5b0",
            "5f85060156194c098868968103dd7df2",
            "fc5d653aa07a4395a7fc4a237ae58431",
            "0fe3d299cf8f4005b53b4b9542471063",
            "9dd1fa7632174187b0beabe7ec780426",
            "7651211a07aa460fa8e854b66527feb0",
            "4d8353f8c93644809e4bbe3cdac8021c",
            "32012cf6be91420882de7bf1ea4955f7",
            "151e70c8af4d4206b1f2b693aecc8f6a"
          ]
        },
        "outputId": "c3871fb8-dba1-4adb-f28e-60412e0c476a"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "\n",
        "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset imdb_reviews/plain_text/1.0.0 (download: 80.23 MiB, generated: Unknown size, total: 80.23 MiB) to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0...\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5a60f634ed9421d9f525b72dbaf76bc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Completed...', max=1.0, style=Progre…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b30ea83194ef453691296fcf9bd8ec07",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Size...', max=1.0, style=ProgressSty…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a446cdf11d9d4e228ba74f3923733dcd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incompleteIN04ID/imdb_reviews-train.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9aecbee6667a4b7f96de3ef0f5cbac15",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=25000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "48ed7fa60135403ab22797f7456e801c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incompleteIN04ID/imdb_reviews-test.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3682132f5ea14c0885f224324793b108",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=25000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "58781d9b89cb416fbdf931f055cd3669",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0.incompleteIN04ID/imdb_reviews-unsupervised.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5f85060156194c098868968103dd7df2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=50000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDataset imdb_reviews downloaded and prepared to /root/tensorflow_datasets/imdb_reviews/plain_text/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n",
            "\r"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6gzSWjYcKADl",
        "colab": {}
      },
      "source": [
        "train_size = info.splits[\"train\"].num_examples\n",
        "test_size = info.splits[\"test\"].num_examples"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4_mhwb7DMHsa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2bde7900-1124-434a-eca5-0b28a8d9b9ea"
      },
      "source": [
        "train_size, test_size"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(25000, 25000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2R_BcFyvMKKT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "cadd11b9-d56c-4678-da51-c3a39a60e2d0"
      },
      "source": [
        "datasets.keys()"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['test', 'train', 'unsupervised'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CiwQDMVs3KEg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "outputId": "f1494447-46ec-48c5-e9de-a41259208f08"
      },
      "source": [
        "for X_batch, y_batch in datasets[\"train\"].batch(2).take(1):\n",
        "    for review, label in zip(X_batch.numpy(), y_batch.numpy()):\n",
        "        print(\"Review:\", review.decode(\"utf-8\")[:200], \"...\")\n",
        "        print(\"Label:\", label, \"= Positive\" if label else \"= Negative\")\n",
        "        print()"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: This was an absolutely terrible movie. Don't be lured in by Christopher Walken or Michael Ironside. Both are great actors, but this must simply be their worst role in history. Even their great acting  ...\n",
            "Label: 0 = Negative\n",
            "\n",
            "Review: I have been known to fall asleep during films, but this is usually due to a combination of things including, really tired, being warm and comfortable on the sette and having just eaten a lot. However  ...\n",
            "Label: 0 = Negative\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kQvPc7-E31nh",
        "colab": {}
      },
      "source": [
        "def preprocess(X_batch, y_batch):\n",
        "    X_batch = tf.strings.substr(X_batch, 0, 300)\n",
        "    X_batch = tf.strings.regex_replace(X_batch, rb\"<br\\s*/?>\", b\" \")\n",
        "    X_batch = tf.strings.regex_replace(X_batch, b\"[^a-zA-Z']\", b\" \")\n",
        "    X_batch = tf.strings.split(X_batch)\n",
        "    return X_batch.to_tensor(default_value=b\"<pad>\"), y_batch"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qFD7_nXT5ykR",
        "colab": {}
      },
      "source": [
        "from collections import Counter\n",
        "vocabulary = Counter()\n",
        "for X_batch, y_batch in datasets[\"train\"].batch(32).map(preprocess):\n",
        "    for review in X_batch:\n",
        "        vocabulary.update(list(review.numpy()))"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2YWQnnvV6I1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d493b84a-4dc5-4492-defe-f62d8f7f05a7"
      },
      "source": [
        "vocabulary.most_common()[:3]"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(b'<pad>', 214309), (b'the', 61137), (b'a', 38564)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OBG33psU6OtF",
        "colab": {}
      },
      "source": [
        "vocab_size = 10000\n",
        "truncated_vocabulary = [\n",
        "    word for word, count in vocabulary.most_common()[:vocab_size]]"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YVhRqp3g6Tvy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cf870727-47f5-4c29-b6ba-9b772703f7af"
      },
      "source": [
        "truncated_vocabulary"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[b'<pad>',\n",
              " b'the',\n",
              " b'a',\n",
              " b'of',\n",
              " b'and',\n",
              " b'to',\n",
              " b'I',\n",
              " b'is',\n",
              " b'in',\n",
              " b'this',\n",
              " b'it',\n",
              " b'was',\n",
              " b'movie',\n",
              " b'that',\n",
              " b'The',\n",
              " b'film',\n",
              " b'with',\n",
              " b'for',\n",
              " b'as',\n",
              " b'on',\n",
              " b'but',\n",
              " b'have',\n",
              " b'This',\n",
              " b'one',\n",
              " b'not',\n",
              " b'be',\n",
              " b'are',\n",
              " b'you',\n",
              " b'an',\n",
              " b'at',\n",
              " b'about',\n",
              " b'by',\n",
              " b'all',\n",
              " b'his',\n",
              " b'so',\n",
              " b'like',\n",
              " b'from',\n",
              " b'who',\n",
              " b'has',\n",
              " b'It',\n",
              " b'good',\n",
              " b'my',\n",
              " b'just',\n",
              " b'very',\n",
              " b'out',\n",
              " b'or',\n",
              " b'story',\n",
              " b'some',\n",
              " b'time',\n",
              " b'had',\n",
              " b'he',\n",
              " b'they',\n",
              " b'really',\n",
              " b'me',\n",
              " b'when',\n",
              " b'what',\n",
              " b'first',\n",
              " b'movies',\n",
              " b'bad',\n",
              " b'see',\n",
              " b'seen',\n",
              " b'up',\n",
              " b'only',\n",
              " b'were',\n",
              " b\"it's\",\n",
              " b'would',\n",
              " b'more',\n",
              " b'made',\n",
              " b'great',\n",
              " b'can',\n",
              " b'been',\n",
              " b'i',\n",
              " b'her',\n",
              " b'no',\n",
              " b'A',\n",
              " b'which',\n",
              " b'even',\n",
              " b'films',\n",
              " b'there',\n",
              " b'ever',\n",
              " b'people',\n",
              " b'much',\n",
              " b'because',\n",
              " b'most',\n",
              " b'plot',\n",
              " b'if',\n",
              " b'than',\n",
              " b'acting',\n",
              " b'get',\n",
              " b'their',\n",
              " b'well',\n",
              " b'into',\n",
              " b'how',\n",
              " b'best',\n",
              " b'think',\n",
              " b'other',\n",
              " b'its',\n",
              " b\"It's\",\n",
              " b'saw',\n",
              " b'could',\n",
              " b'watch',\n",
              " b'many',\n",
              " b\"don't\",\n",
              " b'do',\n",
              " b'will',\n",
              " b'say',\n",
              " b'show',\n",
              " b'love',\n",
              " b'make',\n",
              " b'two',\n",
              " b'way',\n",
              " b'know',\n",
              " b'years',\n",
              " b'after',\n",
              " b'But',\n",
              " b'we',\n",
              " b'watching',\n",
              " b'In',\n",
              " b'never',\n",
              " b'any',\n",
              " b'life',\n",
              " b'characters',\n",
              " b\"I'm\",\n",
              " b'too',\n",
              " b'did',\n",
              " b\"I've\",\n",
              " b'little',\n",
              " b'over',\n",
              " b'If',\n",
              " b'she',\n",
              " b'then',\n",
              " b'being',\n",
              " b'thought',\n",
              " b'them',\n",
              " b'off',\n",
              " b'There',\n",
              " b's',\n",
              " b'funny',\n",
              " b'better',\n",
              " b'also',\n",
              " b'watched',\n",
              " b'where',\n",
              " b'still',\n",
              " b'worst',\n",
              " b'old',\n",
              " b'am',\n",
              " b'through',\n",
              " b'actors',\n",
              " b'And',\n",
              " b'character',\n",
              " b'such',\n",
              " b\"didn't\",\n",
              " b'back',\n",
              " b'got',\n",
              " b'horror',\n",
              " b'those',\n",
              " b'series',\n",
              " b'lot',\n",
              " b'should',\n",
              " b'comedy',\n",
              " b'go',\n",
              " b'your',\n",
              " b'few',\n",
              " b'something',\n",
              " b'does',\n",
              " b'thing',\n",
              " b'him',\n",
              " b'man',\n",
              " b'What',\n",
              " b'going',\n",
              " b'original',\n",
              " b'director',\n",
              " b'work',\n",
              " b'here',\n",
              " b'before',\n",
              " b'real',\n",
              " b'actually',\n",
              " b'cast',\n",
              " b'pretty',\n",
              " b'TV',\n",
              " b'quite',\n",
              " b'why',\n",
              " b'every',\n",
              " b'He',\n",
              " b'find',\n",
              " b'As',\n",
              " b'scenes',\n",
              " b'young',\n",
              " b\"can't\",\n",
              " b'interesting',\n",
              " b'same',\n",
              " b'fan',\n",
              " b'now',\n",
              " b'makes',\n",
              " b'end',\n",
              " b'big',\n",
              " b'When',\n",
              " b'long',\n",
              " b'nothing',\n",
              " b'times',\n",
              " b'found',\n",
              " b\"'s\",\n",
              " b'must',\n",
              " b'these',\n",
              " b'again',\n",
              " b'another',\n",
              " b'part',\n",
              " b'action',\n",
              " b'though',\n",
              " b'always',\n",
              " b'script',\n",
              " b\"doesn't\",\n",
              " b'last',\n",
              " b'new',\n",
              " b'fact',\n",
              " b'year',\n",
              " b'while',\n",
              " b'around',\n",
              " b'look',\n",
              " b'since',\n",
              " b't',\n",
              " b'minutes',\n",
              " b'family',\n",
              " b'DVD',\n",
              " b'down',\n",
              " b'scene',\n",
              " b'things',\n",
              " b'want',\n",
              " b'read',\n",
              " b'seems',\n",
              " b'My',\n",
              " b'fun',\n",
              " b'may',\n",
              " b'world',\n",
              " b'done',\n",
              " b'far',\n",
              " b'come',\n",
              " b'right',\n",
              " b'take',\n",
              " b\"wasn't\",\n",
              " b'You',\n",
              " b'th',\n",
              " b'believe',\n",
              " b'almost',\n",
              " b'enough',\n",
              " b'hard',\n",
              " b'whole',\n",
              " b'anything',\n",
              " b'book',\n",
              " b\"isn't\",\n",
              " b'probably',\n",
              " b'least',\n",
              " b'give',\n",
              " b'bit',\n",
              " b'They',\n",
              " b'American',\n",
              " b'seeing',\n",
              " b'without',\n",
              " b'feel',\n",
              " b'One',\n",
              " b'awful',\n",
              " b'idea',\n",
              " b'making',\n",
              " b'between',\n",
              " b'remember',\n",
              " b'came',\n",
              " b'music',\n",
              " b'kind',\n",
              " b'set',\n",
              " b'excellent',\n",
              " b'John',\n",
              " b'classic',\n",
              " b'us',\n",
              " b'both',\n",
              " b'Well',\n",
              " b'version',\n",
              " b'performance',\n",
              " b'gets',\n",
              " b'plays',\n",
              " b'day',\n",
              " b'Not',\n",
              " b'low',\n",
              " b'might',\n",
              " b'own',\n",
              " b'loved',\n",
              " b'budget',\n",
              " b'role',\n",
              " b'reason',\n",
              " b'high',\n",
              " b'First',\n",
              " b'girl',\n",
              " b'shows',\n",
              " b'looking',\n",
              " b'After',\n",
              " b'For',\n",
              " b'So',\n",
              " b'video',\n",
              " b'special',\n",
              " b'However',\n",
              " b'effects',\n",
              " b'woman',\n",
              " b'different',\n",
              " b'terrible',\n",
              " b'true',\n",
              " b'point',\n",
              " b'start',\n",
              " b'actor',\n",
              " b'put',\n",
              " b'night',\n",
              " b'three',\n",
              " b'played',\n",
              " b'liked',\n",
              " b'yet',\n",
              " b'boring',\n",
              " b'beautiful',\n",
              " b'takes',\n",
              " b'especially',\n",
              " b'based',\n",
              " b'ago',\n",
              " b'main',\n",
              " b'short',\n",
              " b'play',\n",
              " b'guy',\n",
              " b'title',\n",
              " b'said',\n",
              " b'enjoyed',\n",
              " b'anyone',\n",
              " b'wonderful',\n",
              " b'trying',\n",
              " b\"that's\",\n",
              " b'sure',\n",
              " b'went',\n",
              " b'favorite',\n",
              " b'having',\n",
              " b'stars',\n",
              " b'together',\n",
              " b'Hollywood',\n",
              " b'absolutely',\n",
              " b'screen',\n",
              " b'our',\n",
              " b'wrong',\n",
              " b'rather',\n",
              " b'half',\n",
              " b'production',\n",
              " b'episode',\n",
              " b'goes',\n",
              " b'place',\n",
              " b'comes',\n",
              " b'early',\n",
              " b'piece',\n",
              " b'written',\n",
              " b'We',\n",
              " b'heard',\n",
              " b'away',\n",
              " b\"couldn't\",\n",
              " b'second',\n",
              " b'money',\n",
              " b'during',\n",
              " b'friends',\n",
              " b'job',\n",
              " b'THE',\n",
              " b'someone',\n",
              " b'entertaining',\n",
              " b'sense',\n",
              " b'looks',\n",
              " b'She',\n",
              " b'line',\n",
              " b'school',\n",
              " b'drama',\n",
              " b'each',\n",
              " b'flick',\n",
              " b'everything',\n",
              " b'understand',\n",
              " b'No',\n",
              " b'mean',\n",
              " b'course',\n",
              " b'enjoy',\n",
              " b'star',\n",
              " b'wanted',\n",
              " b'wife',\n",
              " b'directed',\n",
              " b'That',\n",
              " b'To',\n",
              " b'small',\n",
              " b'mind',\n",
              " b'seem',\n",
              " b'truly',\n",
              " b'While',\n",
              " b'stupid',\n",
              " b'simply',\n",
              " b\"'\",\n",
              " b'poor',\n",
              " b'All',\n",
              " b'worth',\n",
              " b'everyone',\n",
              " b'once',\n",
              " b'worse',\n",
              " b'top',\n",
              " b'completely',\n",
              " b'used',\n",
              " b'cinema',\n",
              " b'felt',\n",
              " b'kids',\n",
              " b'genre',\n",
              " b'shot',\n",
              " b'home',\n",
              " b'name',\n",
              " b'late',\n",
              " b'tell',\n",
              " b'called',\n",
              " b'until',\n",
              " b'reviews',\n",
              " b'New',\n",
              " b'myself',\n",
              " b'beginning',\n",
              " b'full',\n",
              " b'nice',\n",
              " b'live',\n",
              " b'left',\n",
              " b'At',\n",
              " b'horrible',\n",
              " b'novel',\n",
              " b'How',\n",
              " b'Although',\n",
              " b'thriller',\n",
              " b'documentary',\n",
              " b'women',\n",
              " b'starts',\n",
              " b'performances',\n",
              " b'several',\n",
              " b'else',\n",
              " b'Even',\n",
              " b'audience',\n",
              " b'less',\n",
              " b'perfect',\n",
              " b'often',\n",
              " b'couple',\n",
              " b'writing',\n",
              " b'amazing',\n",
              " b\"I'd\",\n",
              " b'review',\n",
              " b'released',\n",
              " b'seemed',\n",
              " b'history',\n",
              " b'need',\n",
              " b'case',\n",
              " b'Some',\n",
              " b'started',\n",
              " b'later',\n",
              " b'war',\n",
              " b'waste',\n",
              " b'Michael',\n",
              " b'try',\n",
              " b'use',\n",
              " b'either',\n",
              " b'given',\n",
              " b'humor',\n",
              " b'comments',\n",
              " b'B',\n",
              " b'black',\n",
              " b'disappointed',\n",
              " b'although',\n",
              " b'laugh',\n",
              " b'father',\n",
              " b'along',\n",
              " b'gave',\n",
              " b'however',\n",
              " b'quality',\n",
              " b'b',\n",
              " b'expect',\n",
              " b'totally',\n",
              " b'friend',\n",
              " b'gives',\n",
              " b'OK',\n",
              " b'itself',\n",
              " b'sex',\n",
              " b'thinking',\n",
              " b'surprised',\n",
              " b\"you're\",\n",
              " b'brilliant',\n",
              " b'With',\n",
              " b'Now',\n",
              " b'others',\n",
              " b'definitely',\n",
              " b'entire',\n",
              " b'dialogue',\n",
              " b'days',\n",
              " b'ending',\n",
              " b\"he's\",\n",
              " b'television',\n",
              " b'already',\n",
              " b'getting',\n",
              " b'past',\n",
              " b'let',\n",
              " b'James',\n",
              " b'boy',\n",
              " b'camera',\n",
              " b'style',\n",
              " b'problem',\n",
              " b'certainly',\n",
              " b'Just',\n",
              " b'group',\n",
              " b'stories',\n",
              " b'lead',\n",
              " b'o',\n",
              " b'town',\n",
              " b'Why',\n",
              " b'Of',\n",
              " b'maybe',\n",
              " b'person',\n",
              " b'himself',\n",
              " b'playing',\n",
              " b'lost',\n",
              " b'art',\n",
              " b'supposed',\n",
              " b'become',\n",
              " b'doing',\n",
              " b'usually',\n",
              " b'lives',\n",
              " b'men',\n",
              " b'looked',\n",
              " b'moments',\n",
              " b'death',\n",
              " b'hit',\n",
              " b'kid',\n",
              " b'took',\n",
              " b'slow',\n",
              " b'house',\n",
              " b'known',\n",
              " b'sort',\n",
              " b'fine',\n",
              " b'example',\n",
              " b'recently',\n",
              " b'picture',\n",
              " b'premise',\n",
              " b'An',\n",
              " b'His',\n",
              " b'crap',\n",
              " b'knew',\n",
              " b'British',\n",
              " b'direction',\n",
              " b'Robert',\n",
              " b'child',\n",
              " b'decided',\n",
              " b'hour',\n",
              " b'parts',\n",
              " b'Mr',\n",
              " b'help',\n",
              " b'huge',\n",
              " b'expecting',\n",
              " b'From',\n",
              " b'guess',\n",
              " b'fans',\n",
              " b'lines',\n",
              " b'Man',\n",
              " b'works',\n",
              " b'son',\n",
              " b'human',\n",
              " b'attempt',\n",
              " b'David',\n",
              " b'children',\n",
              " b'decent',\n",
              " b'comment',\n",
              " b'extremely',\n",
              " b'finally',\n",
              " b'sequel',\n",
              " b'jokes',\n",
              " b'keep',\n",
              " b'admit',\n",
              " b'actress',\n",
              " b'etc',\n",
              " b'writer',\n",
              " b\"there's\",\n",
              " b'episodes',\n",
              " b'against',\n",
              " b'type',\n",
              " b'game',\n",
              " b'simple',\n",
              " b'musical',\n",
              " b'heart',\n",
              " b'George',\n",
              " b'turned',\n",
              " b'Yes',\n",
              " b'tale',\n",
              " b'storyline',\n",
              " b'under',\n",
              " b'greatest',\n",
              " b'age',\n",
              " b'romantic',\n",
              " b'run',\n",
              " b'opinion',\n",
              " b'rest',\n",
              " b'hilarious',\n",
              " b'Richard',\n",
              " b'able',\n",
              " b'perhaps',\n",
              " b'expected',\n",
              " b'w',\n",
              " b'saying',\n",
              " b'close',\n",
              " b'theater',\n",
              " b'act',\n",
              " b'opening',\n",
              " b'involved',\n",
              " b'killer',\n",
              " b'told',\n",
              " b'typical',\n",
              " b\"I'll\",\n",
              " b'guys',\n",
              " b'Film',\n",
              " b'throughout',\n",
              " b'scary',\n",
              " b'face',\n",
              " b're',\n",
              " b'head',\n",
              " b'turn',\n",
              " b'usual',\n",
              " b'cannot',\n",
              " b'mother',\n",
              " b'serious',\n",
              " b'hours',\n",
              " b'York',\n",
              " b'Its',\n",
              " b'ridiculous',\n",
              " b'bunch',\n",
              " b'feeling',\n",
              " b'next',\n",
              " b'dark',\n",
              " b\"Don't\",\n",
              " b'wants',\n",
              " b'care',\n",
              " b'Unfortunately',\n",
              " b'On',\n",
              " b'predictable',\n",
              " b'word',\n",
              " b'side',\n",
              " b'chance',\n",
              " b'tries',\n",
              " b'car',\n",
              " b'Then',\n",
              " b'fi',\n",
              " b'today',\n",
              " b'Disney',\n",
              " b\"There's\",\n",
              " b'remake',\n",
              " b'self',\n",
              " b'rented',\n",
              " b'sometimes',\n",
              " b'Peter',\n",
              " b'br',\n",
              " b'class',\n",
              " b'dead',\n",
              " b'strange',\n",
              " b'local',\n",
              " b'across',\n",
              " b'acted',\n",
              " b'girls',\n",
              " b'cool',\n",
              " b'animation',\n",
              " b'middle',\n",
              " b'happened',\n",
              " b'silly',\n",
              " b'rating',\n",
              " b'stop',\n",
              " b'instead',\n",
              " b'modern',\n",
              " b\"haven't\",\n",
              " b'interest',\n",
              " b'sound',\n",
              " b'murder',\n",
              " b'doubt',\n",
              " b'matter',\n",
              " b'complete',\n",
              " b'exactly',\n",
              " b'fantastic',\n",
              " b'experience',\n",
              " b\"won't\",\n",
              " b'release',\n",
              " b'talent',\n",
              " b'stuff',\n",
              " b'living',\n",
              " b'white',\n",
              " b'IMDb',\n",
              " b'single',\n",
              " b'obviously',\n",
              " b'Tom',\n",
              " b'upon',\n",
              " b'enjoyable',\n",
              " b'four',\n",
              " b'five',\n",
              " b'except',\n",
              " b'S',\n",
              " b'tried',\n",
              " b'bought',\n",
              " b'reading',\n",
              " b'highly',\n",
              " b'm',\n",
              " b'coming',\n",
              " b'due',\n",
              " b'behind',\n",
              " b'call',\n",
              " b'Most',\n",
              " b'e',\n",
              " b'brother',\n",
              " b'eyes',\n",
              " b'French',\n",
              " b'wonder',\n",
              " b'War',\n",
              " b'number',\n",
              " b'shown',\n",
              " b'sad',\n",
              " b'tells',\n",
              " b'famous',\n",
              " b'feature',\n",
              " b'filmed',\n",
              " b'strong',\n",
              " b'becomes',\n",
              " b'somewhat',\n",
              " b'box',\n",
              " b'Oh',\n",
              " b'hope',\n",
              " b'wrote',\n",
              " b'season',\n",
              " b'obvious',\n",
              " b'happen',\n",
              " b'adaptation',\n",
              " b'clich',\n",
              " b'Paul',\n",
              " b'lame',\n",
              " b'Like',\n",
              " b'viewing',\n",
              " b'poorly',\n",
              " b'moving',\n",
              " b'career',\n",
              " b'evil',\n",
              " b'recommend',\n",
              " b'King',\n",
              " b'husband',\n",
              " b'Oscar',\n",
              " b'named',\n",
              " b'viewer',\n",
              " b'begin',\n",
              " b'violence',\n",
              " b'write',\n",
              " b'non',\n",
              " b'ones',\n",
              " b'view',\n",
              " b'expectations',\n",
              " b'among',\n",
              " b'dull',\n",
              " b'wish',\n",
              " b'masterpiece',\n",
              " b'major',\n",
              " b'straight',\n",
              " b'agree',\n",
              " b'Movie',\n",
              " b'sit',\n",
              " b'hand',\n",
              " b'period',\n",
              " b'directing',\n",
              " b'particularly',\n",
              " b'country',\n",
              " b'fight',\n",
              " b'hate',\n",
              " b'daughter',\n",
              " b'happens',\n",
              " b'English',\n",
              " b'says',\n",
              " b'stand',\n",
              " b'turns',\n",
              " b'Christmas',\n",
              " b'f',\n",
              " b'attention',\n",
              " b\"That's\",\n",
              " b'taken',\n",
              " b'cheesy',\n",
              " b'comic',\n",
              " b'mostly',\n",
              " b\"wouldn't\",\n",
              " b'light',\n",
              " b'entertainment',\n",
              " b'order',\n",
              " b'roles',\n",
              " b'superb',\n",
              " b'dialog',\n",
              " b'easily',\n",
              " b'starring',\n",
              " b'Lee',\n",
              " b'weak',\n",
              " b'mystery',\n",
              " b'Night',\n",
              " b'seriously',\n",
              " b'realistic',\n",
              " b'reasons',\n",
              " b'theme',\n",
              " b'possibly',\n",
              " b'words',\n",
              " b'ten',\n",
              " b'cheap',\n",
              " b'similar',\n",
              " b'brought',\n",
              " b'co',\n",
              " b'including',\n",
              " b'nearly',\n",
              " b'cinematography',\n",
              " b'falls',\n",
              " b'basically',\n",
              " b'produced',\n",
              " b'talking',\n",
              " b'working',\n",
              " b'taking',\n",
              " b'reality',\n",
              " b'store',\n",
              " b'unique',\n",
              " b'forward',\n",
              " b'hell',\n",
              " b'alone',\n",
              " b'T',\n",
              " b'whose',\n",
              " b'sets',\n",
              " b'previous',\n",
              " b'soon',\n",
              " b'themselves',\n",
              " b'lack',\n",
              " b'killed',\n",
              " b'directors',\n",
              " b'c',\n",
              " b'bring',\n",
              " b'week',\n",
              " b'effort',\n",
              " b'average',\n",
              " b'ways',\n",
              " b'female',\n",
              " b'running',\n",
              " b'fast',\n",
              " b'editing',\n",
              " b'kept',\n",
              " b'caught',\n",
              " b'Japanese',\n",
              " b'popular',\n",
              " b'screenplay',\n",
              " b'elements',\n",
              " b'comedies',\n",
              " b'surprise',\n",
              " b'became',\n",
              " b'problems',\n",
              " b'Jack',\n",
              " b'features',\n",
              " b'annoying',\n",
              " b'animated',\n",
              " b'yes',\n",
              " b'd',\n",
              " b'possible',\n",
              " b'cover',\n",
              " b'follow',\n",
              " b'near',\n",
              " b'William',\n",
              " b'list',\n",
              " b'songs',\n",
              " b'cartoon',\n",
              " b'Having',\n",
              " b'voice',\n",
              " b'parents',\n",
              " b'Maybe',\n",
              " b'Star',\n",
              " b'events',\n",
              " b'none',\n",
              " b'body',\n",
              " b'crime',\n",
              " b'h',\n",
              " b'Great',\n",
              " b'Who',\n",
              " b'future',\n",
              " b'Very',\n",
              " b'weird',\n",
              " b'God',\n",
              " b\"aren't\",\n",
              " b'actual',\n",
              " b'despite',\n",
              " b'final',\n",
              " b'Is',\n",
              " b'gore',\n",
              " b'imagine',\n",
              " b'change',\n",
              " b'sister',\n",
              " b'cut',\n",
              " b\"you'll\",\n",
              " b'D',\n",
              " b'lots',\n",
              " b'beyond',\n",
              " b'moment',\n",
              " b'Dr',\n",
              " b'positive',\n",
              " b'level',\n",
              " b'fantasy',\n",
              " b'important',\n",
              " b'minute',\n",
              " b'sci',\n",
              " b'Joe',\n",
              " b'easy',\n",
              " b'era',\n",
              " b'House',\n",
              " b'awesome',\n",
              " b'romance',\n",
              " b'above',\n",
              " b'team',\n",
              " b'subject',\n",
              " b'rent',\n",
              " b'badly',\n",
              " b'happy',\n",
              " b'finds',\n",
              " b'points',\n",
              " b'rate',\n",
              " b\"'The\",\n",
              " b'begins',\n",
              " b'message',\n",
              " b'incredibly',\n",
              " b'concept',\n",
              " b'Italian',\n",
              " b'realize',\n",
              " b'Good',\n",
              " b'recent',\n",
              " b'score',\n",
              " b'By',\n",
              " b'casting',\n",
              " b'America',\n",
              " b'police',\n",
              " b'SPOILERS',\n",
              " b'power',\n",
              " b\"you've\",\n",
              " b'NOT',\n",
              " b\"they're\",\n",
              " b'giving',\n",
              " b'college',\n",
              " b'difficult',\n",
              " b'fall',\n",
              " b'laughed',\n",
              " b'knows',\n",
              " b'save',\n",
              " b'incredible',\n",
              " b'third',\n",
              " b'potential',\n",
              " b'leave',\n",
              " b'song',\n",
              " b'clearly',\n",
              " b'Scott',\n",
              " b'slasher',\n",
              " b'talk',\n",
              " b'laughing',\n",
              " b'Every',\n",
              " b'meets',\n",
              " b'move',\n",
              " b'fairly',\n",
              " b'kill',\n",
              " b'hero',\n",
              " b'plain',\n",
              " b'means',\n",
              " b'p',\n",
              " b'adventure',\n",
              " b'suspense',\n",
              " b'certain',\n",
              " b'joke',\n",
              " b'material',\n",
              " b'rated',\n",
              " b'showing',\n",
              " b'Also',\n",
              " b'mess',\n",
              " b'needs',\n",
              " b'World',\n",
              " b'Ben',\n",
              " b'gone',\n",
              " b'books',\n",
              " b'disappointment',\n",
              " b'cop',\n",
              " b'talented',\n",
              " b'copy',\n",
              " b'Bill',\n",
              " b'Festival',\n",
              " b'showed',\n",
              " b'success',\n",
              " b'interested',\n",
              " b'fails',\n",
              " b'shots',\n",
              " b'stage',\n",
              " b'describe',\n",
              " b'Wow',\n",
              " b'Okay',\n",
              " b\"she's\",\n",
              " b'wasted',\n",
              " b'Jane',\n",
              " b'particular',\n",
              " b'spent',\n",
              " b'whether',\n",
              " b'bored',\n",
              " b'wait',\n",
              " b'clear',\n",
              " b'rare',\n",
              " b'Jim',\n",
              " b'trash',\n",
              " b'consider',\n",
              " ...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "8xb3T_4D69zP"
      },
      "source": [
        "Now we need to add a preprocessing step to replace each word with its ID (i.e., its index in the vocabulary)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "K91mBcZi6a9v",
        "colab": {}
      },
      "source": [
        "words = tf.constant(truncated_vocabulary)\n",
        "word_ids = tf.range(len(truncated_vocabulary), dtype=tf.int64)\n",
        "vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n",
        "num_oov_buckets = 1000\n",
        "table = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)"
      ],
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6lTpmmQv7VsH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d0795680-fafd-4592-a6dd-41adfe7e50a9"
      },
      "source": [
        "table.lookup(tf.constant([b\"This movie was awesome\".split()]))"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 4), dtype=int64, numpy=array([[ 22,  12,  11, 902]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "49spwXIN7a2Z",
        "colab": {}
      },
      "source": [
        "def encode_words(X_batch, y_batch):\n",
        "    return table.lookup(X_batch), y_batch"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "r13L_K7E7qpr",
        "colab": {}
      },
      "source": [
        "train_set = datasets[\"train\"].batch(32).map(preprocess)\n",
        "train_set = train_set.map(encode_words).prefetch(1)"
      ],
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9QFB-I5T7zNg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "77b1c137-5bf2-4457-9d08-368ff46d9cd4"
      },
      "source": [
        "for X_batch, y_batch in train_set.take(1):\n",
        "    print(X_batch)\n",
        "    print(y_batch)"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[[  22   11   28 ...    0    0    0]\n",
            " [   6   21   70 ...    0    0    0]\n",
            " [4099 6881    1 ...    0    0    0]\n",
            " ...\n",
            " [  22   12  118 ...  331 1047    0]\n",
            " [1757 4101  451 ...    0    0    0]\n",
            " [3365 4392    6 ...    0    0    0]], shape=(32, 60), dtype=int64)\n",
            "tf.Tensor([0 0 0 1 1 1 0 0 0 0 0 1 1 0 1 0 1 1 1 0 1 1 1 1 1 0 0 0 1 0 0 0], shape=(32,), dtype=int64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K8oAplmOB9vL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_set = datasets[\"test\"].batch(32).map(preprocess)\n",
        "test_set = test_set.map(encode_words).prefetch(1)"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "peUnpjxM72iK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        },
        "outputId": "74198820-600d-4e08-909f-a0b4c12cbb38"
      },
      "source": [
        "embed_size = 128\n",
        "model = keras.models.Sequential([\n",
        "    keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n",
        "                           mask_zero=True, # not shown in the book\n",
        "                           input_shape=[None]),\n",
        "    keras.layers.GRU(128, return_sequences=True),\n",
        "    keras.layers.GRU(128),\n",
        "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
        "])\n",
        "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "history = model.fit(train_set, epochs=10, validation_data = test_set, callbacks = [NvidiaUtilizationCallback()])"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%  GPU utilization\n",
            "Epoch 1/10\n",
            "782/782 [==============================] - 58s 74ms/step - loss: 0.5305 - accuracy: 0.7281 - val_loss: 0.4721 - val_accuracy: 0.7737\n",
            "  0%  GPU utilization\n",
            "Epoch 2/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.3459 - accuracy: 0.8549 - val_loss: 0.5153 - val_accuracy: 0.7608\n",
            "  0%  GPU utilization\n",
            "Epoch 3/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.1934 - accuracy: 0.9313 - val_loss: 0.6230 - val_accuracy: 0.7481\n",
            "  0%  GPU utilization\n",
            "Epoch 4/10\n",
            "782/782 [==============================] - 56s 71ms/step - loss: 0.1361 - accuracy: 0.9504 - val_loss: 0.7142 - val_accuracy: 0.7399\n",
            "  0%  GPU utilization\n",
            "Epoch 5/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.1032 - accuracy: 0.9634 - val_loss: 0.9483 - val_accuracy: 0.7166\n",
            "  0%  GPU utilization\n",
            "Epoch 6/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.0716 - accuracy: 0.9743 - val_loss: 0.9481 - val_accuracy: 0.7301\n",
            "  0%  GPU utilization\n",
            "Epoch 7/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.0587 - accuracy: 0.9799 - val_loss: 1.1257 - val_accuracy: 0.7330\n",
            "  0%  GPU utilization\n",
            "Epoch 8/10\n",
            "782/782 [==============================] - 55s 71ms/step - loss: 0.0412 - accuracy: 0.9854 - val_loss: 1.1625 - val_accuracy: 0.7300\n",
            "  0%  GPU utilization\n",
            "Epoch 9/10\n",
            "782/782 [==============================] - 56s 71ms/step - loss: 0.0380 - accuracy: 0.9874 - val_loss: 1.2573 - val_accuracy: 0.7282\n",
            "  0%  GPU utilization\n",
            "Epoch 10/10\n",
            "782/782 [==============================] - 55s 70ms/step - loss: 0.0311 - accuracy: 0.9897 - val_loss: 1.4680 - val_accuracy: 0.7393\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YF_oQ-ivAwos",
        "colab": {}
      },
      "source": [
        "# Same code as above\n",
        "#------------------------------\n",
        "\n",
        "# K = keras.backend\n",
        "# embed_size = 128\n",
        "# inputs = keras.layers.Input(shape=[None])\n",
        "# mask = keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n",
        "# z = keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n",
        "# z = keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n",
        "# z = keras.layers.GRU(128)(z, mask=mask)\n",
        "# outputs = keras.layers.Dense(1, activation=\"sigmoid\")(z)\n",
        "# model = keras.models.Model(inputs=[inputs], outputs=[outputs])\n",
        "# model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "# history = model.fit(train_set, epochs=10, validation_data = test_set)   "
      ],
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "nrAHYw1gE9k7",
        "colab": {}
      },
      "source": [
        "def giveSentiment(text):\n",
        "    return model.predict(table.lookup(tf.constant([text.split()])))"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n6lexHAXB9vZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "741606b9-8e73-4eee-c881-f2cc638acdfb"
      },
      "source": [
        "giveSentiment(b\"Omelets are to die for!\")"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.9387816]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-qf684fBB9vb",
        "colab_type": "text"
      },
      "source": [
        "## Reusing Pretrained Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mi7xNegsB9vc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "TFHUB_CACHE_DIR = os.path.join(os.curdir, \"my_tfhub_cache\")\n",
        "os.environ[\"TFHUB_CACHE_DIR\"] = TFHUB_CACHE_DIR"
      ],
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mPf7sRTfB9ve",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow_hub as hub"
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXOTofdTB9vh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with tf.device(\"/cpu:0\"):\n",
        "    model = keras.Sequential([\n",
        "        hub.KerasLayer(\"https://tfhub.dev/google/nnlm-en-dim128/2\",  input_shape=[] ,dtype=tf.string),\n",
        "        keras.layers.Dense(128, activation=\"relu\"),\n",
        "        keras.layers.Dense(1, activation=\"sigmoid\")\n",
        "    ])"
      ],
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kon571kBB9vk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "2e01f012-a1f7-4c43-ae32-696511b2d3a1"
      },
      "source": [
        "for dirpath, dirnames, filenames in os.walk(TFHUB_CACHE_DIR):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirpath, filename))"
      ],
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./my_tfhub_cache/29abffb443cb0a0ca9c72e8e3863b76d85028490.descriptor.txt\n",
            "./my_tfhub_cache/29abffb443cb0a0ca9c72e8e3863b76d85028490/saved_model.pb\n",
            "./my_tfhub_cache/29abffb443cb0a0ca9c72e8e3863b76d85028490/variables/variables.data-00000-of-00001\n",
            "./my_tfhub_cache/29abffb443cb0a0ca9c72e8e3863b76d85028490/variables/variables.index\n",
            "./my_tfhub_cache/29abffb443cb0a0ca9c72e8e3863b76d85028490/assets/tokens.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eUoP-VcJB9vm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "\n",
        "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)\n",
        "train_size = info.splits[\"train\"].num_examples\n",
        "batch_size = 32\n",
        "train_set = datasets[\"train\"].batch(batch_size).prefetch(1)\n",
        "test_set = datasets[\"test\"].batch(batch_size).prefetch(1)"
      ],
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "flAvDvK1B9vq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "837f334f-d199-45c7-e028-7e3d19749d5d"
      },
      "source": [
        "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "history = model.fit(train_set, epochs=20, validation_data = test_set, callbacks = [NvidiaUtilizationCallback()])"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%  GPU utilization\n",
            "Epoch 1/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4832 - accuracy: 0.7717 - val_loss: 0.4549 - val_accuracy: 0.7849\n",
            "  0%  GPU utilization\n",
            "Epoch 2/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4435 - accuracy: 0.7925 - val_loss: 0.4515 - val_accuracy: 0.7858\n",
            "  0%  GPU utilization\n",
            "Epoch 3/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4375 - accuracy: 0.7948 - val_loss: 0.4490 - val_accuracy: 0.7862\n",
            "  0%  GPU utilization\n",
            "Epoch 4/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4323 - accuracy: 0.7986 - val_loss: 0.4454 - val_accuracy: 0.7878\n",
            "  0%  GPU utilization\n",
            "Epoch 5/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.4271 - accuracy: 0.8015 - val_loss: 0.4427 - val_accuracy: 0.7892\n",
            "  0%  GPU utilization\n",
            "Epoch 6/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4219 - accuracy: 0.8048 - val_loss: 0.4398 - val_accuracy: 0.7910\n",
            "  0%  GPU utilization\n",
            "Epoch 7/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4169 - accuracy: 0.8084 - val_loss: 0.4380 - val_accuracy: 0.7923\n",
            "  0%  GPU utilization\n",
            "Epoch 8/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.4119 - accuracy: 0.8121 - val_loss: 0.4364 - val_accuracy: 0.7939\n",
            "  0%  GPU utilization\n",
            "Epoch 9/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.4070 - accuracy: 0.8140 - val_loss: 0.4359 - val_accuracy: 0.7938\n",
            "  0%  GPU utilization\n",
            "Epoch 10/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.4024 - accuracy: 0.8168 - val_loss: 0.4353 - val_accuracy: 0.7948\n",
            "  0%  GPU utilization\n",
            "Epoch 11/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3976 - accuracy: 0.8194 - val_loss: 0.4359 - val_accuracy: 0.7952\n",
            "  0%  GPU utilization\n",
            "Epoch 12/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.3930 - accuracy: 0.8223 - val_loss: 0.4366 - val_accuracy: 0.7950\n",
            "  0%  GPU utilization\n",
            "Epoch 13/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3887 - accuracy: 0.8262 - val_loss: 0.4361 - val_accuracy: 0.7961\n",
            "  0%  GPU utilization\n",
            "Epoch 14/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3844 - accuracy: 0.8287 - val_loss: 0.4386 - val_accuracy: 0.7947\n",
            "  0%  GPU utilization\n",
            "Epoch 15/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3803 - accuracy: 0.8308 - val_loss: 0.4384 - val_accuracy: 0.7942\n",
            "  0%  GPU utilization\n",
            "Epoch 16/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3760 - accuracy: 0.8336 - val_loss: 0.4394 - val_accuracy: 0.7936\n",
            "  0%  GPU utilization\n",
            "Epoch 17/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3719 - accuracy: 0.8366 - val_loss: 0.4410 - val_accuracy: 0.7938\n",
            "  0%  GPU utilization\n",
            "Epoch 18/20\n",
            "782/782 [==============================] - 14s 18ms/step - loss: 0.3677 - accuracy: 0.8382 - val_loss: 0.4435 - val_accuracy: 0.7922\n",
            "  0%  GPU utilization\n",
            "Epoch 19/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.3638 - accuracy: 0.8401 - val_loss: 0.4456 - val_accuracy: 0.7919\n",
            "  0%  GPU utilization\n",
            "Epoch 20/20\n",
            "782/782 [==============================] - 15s 19ms/step - loss: 0.3598 - accuracy: 0.8427 - val_loss: 0.4478 - val_accuracy: 0.7908\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sV7dCuOB9vt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def giveSentiment(text):\n",
        "    return model.predict([text])"
      ],
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydWt2BdAB9v1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8a6dc3fc-3e85-4719-8b5f-ece1f9bcdad8"
      },
      "source": [
        "giveSentiment(\"The movie was not good\")"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.1780042]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yuXzuDqNB9v4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "22cb10e7-0b68-4ce7-f153-9f03d8f61cf6"
      },
      "source": [
        "giveSentiment(\"The movies are to die for!\")"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.32158712]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DYHzoGdMB9v5",
        "colab_type": "text"
      },
      "source": [
        "## An Encoder–Decoder Network for Neural Machine Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwkzxQciB9v7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow_addons as tfa"
      ],
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHCqpkv9B9v9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab_size = 10000\n",
        "embed_size = 128"
      ],
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BU0HGhbB9v_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "bcbea638-9fa8-489f-870e-6f18e7dd72bd"
      },
      "source": [
        "try:\n",
        "    encoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n",
        "    decoder_inputs = keras.layers.Input(shape=[None], dtype=np.int32)\n",
        "    sequence_lengths = keras.layers.Input(shape=[], dtype=np.int32)\n",
        "\n",
        "    embeddings = keras.layers.Embedding(vocab_size, embed_size)\n",
        "    encoder_embeddings = embeddings(encoder_inputs)\n",
        "    decoder_embeddings = embeddings(decoder_inputs)\n",
        "\n",
        "    encoder = keras.layers.LSTM(512, return_state=True)\n",
        "    encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n",
        "    encoder_state = [state_h, state_c]\n",
        "\n",
        "    sampler = tfa.seq2seq.sampler.TrainingSampler()\n",
        "\n",
        "    decoder_cell = keras.layers.LSTMCell(512)\n",
        "    output_layer = keras.layers.Dense(vocab_size)\n",
        "    decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler = sampler,\n",
        "                                                     output_layer=output_layer)\n",
        "    final_outputs, final_state, final_sequence_lengths = decoder(\n",
        "        decoder_embeddings, initial_state=encoder_state,\n",
        "        sequence_length=sequence_lengths)\n",
        "    Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n",
        "\n",
        "    model = keras.models.Model(\n",
        "        inputs=[encoder_inputs, decoder_inputs, sequence_lengths],\n",
        "        outputs=[Y_proba])\n",
        "\n",
        "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\")\n",
        "\n",
        "\n",
        "    X = np.random.randint(100, size=10*1000).reshape(1000, 10)\n",
        "    Y = np.random.randint(100, size=15*1000).reshape(1000, 15)\n",
        "    X_decoder = np.c_[np.zeros((1000, 1)), Y[:, :-1]]\n",
        "    seq_lengths = np.full([1000], 15)\n",
        "\n",
        "    history = model.fit([X, X_decoder, seq_lengths], Y, epochs=2)\n",
        "except TypeError as t:\n",
        "    print(t)"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "32/32 [==============================] - 4s 120ms/step - loss: 6.1357\n",
            "Epoch 2/2\n",
            "32/32 [==============================] - 4s 114ms/step - loss: 4.6636\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OC_F6cTxB9wQ",
        "colab_type": "text"
      },
      "source": [
        "## Bidirectional Recurrent Layers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OBZ0zZObB9wR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "outputId": "36ec3e65-4ede-4dd2-ad12-2908f131a20e"
      },
      "source": [
        "model = keras.models.Sequential([\n",
        "    keras.layers.GRU(10, return_sequences = True, input_shape=[None, 10]),\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(10, return_sequences=True))\n",
        "])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_7\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "gru_9 (GRU)                  (None, None, 10)          660       \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, None, 20)          1320      \n",
            "=================================================================\n",
            "Total params: 1,980\n",
            "Trainable params: 1,980\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}