04_intro_to_llms/IntroLLMs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMZV5UwakJTbqgB0ItPwe9k",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bd858574deec4181bb0ba443be9a2088": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b025f1fa9b61436f93515742e324b86f",
              "IPY_MODEL_5073ff96876f4e04b246e466be18c186",
              "IPY_MODEL_0c9e9d1ae5ed4ae4a7e37513d35f9c77"
            ],
            "layout": "IPY_MODEL_b29ed3c3e599460cbd3d693d71f456d2"
          }
        },
        "b025f1fa9b61436f93515742e324b86f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b43c22a0fa94a7690d637dcef8f9e0a",
            "placeholder": "​",
            "style": "IPY_MODEL_51736c335dd349e7bfcc529ada3c6f6a",
            "value": "config.json: 100%"
          }
        },
        "5073ff96876f4e04b246e466be18c186": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4791ebe7b3cc4444afdcbfd8d1b5967a",
            "max": 629,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_93c49acd409c4616b22b28ef971c9dd6",
            "value": 629
          }
        },
        "0c9e9d1ae5ed4ae4a7e37513d35f9c77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4777bd67c1e64692b294830f05c7143d",
            "placeholder": "​",
            "style": "IPY_MODEL_aeb6c3883112462c8ade793f34ebc4da",
            "value": " 629/629 [00:00&lt;00:00, 11.0kB/s]"
          }
        },
        "b29ed3c3e599460cbd3d693d71f456d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b43c22a0fa94a7690d637dcef8f9e0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51736c335dd349e7bfcc529ada3c6f6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4791ebe7b3cc4444afdcbfd8d1b5967a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93c49acd409c4616b22b28ef971c9dd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4777bd67c1e64692b294830f05c7143d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aeb6c3883112462c8ade793f34ebc4da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "82ebac62b0954db1b9fa3e3ae4025a04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_88bc6af07dc244b8bb3ea694ed31d2ae",
              "IPY_MODEL_21364965ba974886ad36c5d9670e7a70",
              "IPY_MODEL_00c33e96a50b4993a9f3387402fbccfe"
            ],
            "layout": "IPY_MODEL_9de7b337c654426fac8f86b97c7ca7dd"
          }
        },
        "88bc6af07dc244b8bb3ea694ed31d2ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d3d454f41f164fc5af66adf8d2caa53c",
            "placeholder": "​",
            "style": "IPY_MODEL_06cd5703c7134fd49a66a259d0154b85",
            "value": "model.safetensors: 100%"
          }
        },
        "21364965ba974886ad36c5d9670e7a70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6131609549c4e7eb89cbb3f430d8aca",
            "max": 267832558,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e5a7799affd54336b99042ae28609630",
            "value": 267832558
          }
        },
        "00c33e96a50b4993a9f3387402fbccfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ee4ce555b264be4becae07722f88a25",
            "placeholder": "​",
            "style": "IPY_MODEL_4e181643cf4e449ca06776ecbba79c5e",
            "value": " 268M/268M [00:02&lt;00:00, 91.1MB/s]"
          }
        },
        "9de7b337c654426fac8f86b97c7ca7dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3d454f41f164fc5af66adf8d2caa53c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06cd5703c7134fd49a66a259d0154b85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f6131609549c4e7eb89cbb3f430d8aca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5a7799affd54336b99042ae28609630": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5ee4ce555b264be4becae07722f88a25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e181643cf4e449ca06776ecbba79c5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "067f49dd461a44fabf2b2fbc48c38ea3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8081fe15a05f4d93a263016d65f7c867",
              "IPY_MODEL_db4f4633a272445fb65b31b81cc13a47",
              "IPY_MODEL_683187173eee4b4f84c42ee1b030041e"
            ],
            "layout": "IPY_MODEL_9b356d4bb2584404b40a53e84ffd08ad"
          }
        },
        "8081fe15a05f4d93a263016d65f7c867": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8844d328728454cabd0c601f1dfac4c",
            "placeholder": "​",
            "style": "IPY_MODEL_999973bfe8554a6b89b2214b39348331",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "db4f4633a272445fb65b31b81cc13a47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80f90f7e6d644d1d8b7df13c9e85225d",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_683ed139b6384e78a7d16d8375640c74",
            "value": 48
          }
        },
        "683187173eee4b4f84c42ee1b030041e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aaca325b53f449519e4193981a4b3865",
            "placeholder": "​",
            "style": "IPY_MODEL_d1a67e3364a04697a740fac83669d9f9",
            "value": " 48.0/48.0 [00:00&lt;00:00, 1.02kB/s]"
          }
        },
        "9b356d4bb2584404b40a53e84ffd08ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8844d328728454cabd0c601f1dfac4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "999973bfe8554a6b89b2214b39348331": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "80f90f7e6d644d1d8b7df13c9e85225d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "683ed139b6384e78a7d16d8375640c74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aaca325b53f449519e4193981a4b3865": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1a67e3364a04697a740fac83669d9f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9489672461b54621bdb8bb24d0eb9934": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2862c54dff8c4f83829e946c69962221",
              "IPY_MODEL_95367cef20634367b463b8f0a95e0503",
              "IPY_MODEL_fac8d31cc40e4c9e80f8ca93a9075077"
            ],
            "layout": "IPY_MODEL_e61474e5f4784cf6ab8a47decc035ae0"
          }
        },
        "2862c54dff8c4f83829e946c69962221": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b28301ee0a554b009b4a04ab5c3bcaf0",
            "placeholder": "​",
            "style": "IPY_MODEL_8a29da2807084308bd8ccf9414150e31",
            "value": "vocab.txt: 100%"
          }
        },
        "95367cef20634367b463b8f0a95e0503": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_545d6ab11dc744099b608face7404d0e",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6c4a6d9277f148c6a575b1b7c6c008f7",
            "value": 231508
          }
        },
        "fac8d31cc40e4c9e80f8ca93a9075077": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cac928cad64b40cb8d0ef32f3731451a",
            "placeholder": "​",
            "style": "IPY_MODEL_a0e7b739e9a44660b0b8b65448d89a93",
            "value": " 232k/232k [00:00&lt;00:00, 3.42MB/s]"
          }
        },
        "e61474e5f4784cf6ab8a47decc035ae0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b28301ee0a554b009b4a04ab5c3bcaf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a29da2807084308bd8ccf9414150e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "545d6ab11dc744099b608face7404d0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c4a6d9277f148c6a575b1b7c6c008f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cac928cad64b40cb8d0ef32f3731451a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0e7b739e9a44660b0b8b65448d89a93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5c584862c1714236be9a4d72eacb7e8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_878ac7474c814c56a1f8e4ca1307b446",
              "IPY_MODEL_61d31a811e704e0b940337009892c774",
              "IPY_MODEL_90d95ae3318b4f4bbda0f5d016347e91"
            ],
            "layout": "IPY_MODEL_c603255a544f47b280fd61e9e45d9204"
          }
        },
        "878ac7474c814c56a1f8e4ca1307b446": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc69805bf372491d931e20940cf99cfc",
            "placeholder": "​",
            "style": "IPY_MODEL_9eb1cc51fc0348069875ce5c52c992b9",
            "value": "config.json: 100%"
          }
        },
        "61d31a811e704e0b940337009892c774": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a36bbfb2408944c39232013f53811f08",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_16994401181f43238336a07a06be119b",
            "value": 570
          }
        },
        "90d95ae3318b4f4bbda0f5d016347e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc6892bac378490abd7f628bca35906c",
            "placeholder": "​",
            "style": "IPY_MODEL_1dcd4ce0732240aab355557623a19d59",
            "value": " 570/570 [00:00&lt;00:00, 34.8kB/s]"
          }
        },
        "c603255a544f47b280fd61e9e45d9204": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc69805bf372491d931e20940cf99cfc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9eb1cc51fc0348069875ce5c52c992b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a36bbfb2408944c39232013f53811f08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16994401181f43238336a07a06be119b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dc6892bac378490abd7f628bca35906c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1dcd4ce0732240aab355557623a19d59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3933d404330d4d02913a7e075819f807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f286b77c68ce47ec9a5177f2dd35ea48",
              "IPY_MODEL_d15d08eb7374438f8edbcc2d78beafc8",
              "IPY_MODEL_a9a334e404fe47da819a7e97d0dbf958"
            ],
            "layout": "IPY_MODEL_ed261fb3eb6b4661a0dba67a3fc9858b"
          }
        },
        "f286b77c68ce47ec9a5177f2dd35ea48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95d2b19696f44d099e530f82ad213328",
            "placeholder": "​",
            "style": "IPY_MODEL_725a53042a724dc0a9a658bd3dd3cb83",
            "value": "model.safetensors: 100%"
          }
        },
        "d15d08eb7374438f8edbcc2d78beafc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4dd55f635a74dbeb0067d1c47769123",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_832adb3112224a51ba9376a50076dd8c",
            "value": 440449768
          }
        },
        "a9a334e404fe47da819a7e97d0dbf958": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c953196990624b0ea79c2a3c74cd1de9",
            "placeholder": "​",
            "style": "IPY_MODEL_46f89e0a5ae84c6c97265c795fae9f88",
            "value": " 440M/440M [00:06&lt;00:00, 17.6MB/s]"
          }
        },
        "ed261fb3eb6b4661a0dba67a3fc9858b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95d2b19696f44d099e530f82ad213328": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "725a53042a724dc0a9a658bd3dd3cb83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e4dd55f635a74dbeb0067d1c47769123": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "832adb3112224a51ba9376a50076dd8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c953196990624b0ea79c2a3c74cd1de9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46f89e0a5ae84c6c97265c795fae9f88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ad748b4c592645cb90d427c9012d7368": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a34cdc4592cf46ad9ccf69caa52363bd",
              "IPY_MODEL_c1eac1886ea84c87bed161c960569d3d",
              "IPY_MODEL_ca50498e237843709d8c44e7ea0e94b5"
            ],
            "layout": "IPY_MODEL_8f4387f0bb3e402ab1ab4d443c693f96"
          }
        },
        "a34cdc4592cf46ad9ccf69caa52363bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47316e9804804c8d9c9b031fa3d86052",
            "placeholder": "​",
            "style": "IPY_MODEL_6cc5bd956c694bc5b63c0c7bba2a29f8",
            "value": "config.json: 100%"
          }
        },
        "c1eac1886ea84c87bed161c960569d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93728bade1714bcabbe90f1b9b95b7c2",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83cea646339b4e74a9582827afdf5753",
            "value": 665
          }
        },
        "ca50498e237843709d8c44e7ea0e94b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c765d253aa974bc59654f0398afb964d",
            "placeholder": "​",
            "style": "IPY_MODEL_3d71478078bc4743a580dc6200cccbd0",
            "value": " 665/665 [00:00&lt;00:00, 11.6kB/s]"
          }
        },
        "8f4387f0bb3e402ab1ab4d443c693f96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47316e9804804c8d9c9b031fa3d86052": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cc5bd956c694bc5b63c0c7bba2a29f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "93728bade1714bcabbe90f1b9b95b7c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83cea646339b4e74a9582827afdf5753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c765d253aa974bc59654f0398afb964d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d71478078bc4743a580dc6200cccbd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "099e1574f4a14c308be2364d0bba2a76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6f822f4ae7734ce09736d50bf17ad99e",
              "IPY_MODEL_93fbd0ce38df488fa725d7c584344e6b",
              "IPY_MODEL_ee56c1c4c2be4086883a95e15adc655d"
            ],
            "layout": "IPY_MODEL_846bb66c196540febf6a7bc32b60dc3a"
          }
        },
        "6f822f4ae7734ce09736d50bf17ad99e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_37dcd77ea3154208be44bfa35a88fa90",
            "placeholder": "​",
            "style": "IPY_MODEL_fdd9ba7947794f4aa0e2782e3b379f61",
            "value": "model.safetensors: 100%"
          }
        },
        "93fbd0ce38df488fa725d7c584344e6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64fa9056ffef4ab3a4e2db085a48cbfb",
            "max": 548105171,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_acee154ece2e4a939f42236ec38de871",
            "value": 548105171
          }
        },
        "ee56c1c4c2be4086883a95e15adc655d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_46acb20f74174029962ecb8142c576fb",
            "placeholder": "​",
            "style": "IPY_MODEL_b58662cfa5fb4406a171135e01d25368",
            "value": " 548M/548M [00:06&lt;00:00, 70.2MB/s]"
          }
        },
        "846bb66c196540febf6a7bc32b60dc3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37dcd77ea3154208be44bfa35a88fa90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdd9ba7947794f4aa0e2782e3b379f61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64fa9056ffef4ab3a4e2db085a48cbfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acee154ece2e4a939f42236ec38de871": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "46acb20f74174029962ecb8142c576fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b58662cfa5fb4406a171135e01d25368": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c770f4a4cad0435ebedbb932c1431f52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_14dab4fee3d34783bf838cfbd4bdbcaa",
              "IPY_MODEL_d00cd656bc4f4f0d9cf52b942fceb155",
              "IPY_MODEL_d5fbc43ad5204bb88b11ddceaedc5d6d"
            ],
            "layout": "IPY_MODEL_8d825e3d51524720ada2f1ecd9a9ffd5"
          }
        },
        "14dab4fee3d34783bf838cfbd4bdbcaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a17934807c140eeb8c0a5cf80c4cb50",
            "placeholder": "​",
            "style": "IPY_MODEL_7d2254be42234d24b044f6c9c0d49119",
            "value": "generation_config.json: 100%"
          }
        },
        "d00cd656bc4f4f0d9cf52b942fceb155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9eddddc3c9034e6da5bf612da55f5714",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9548afdd0d154788ba3edb96c326c85f",
            "value": 124
          }
        },
        "d5fbc43ad5204bb88b11ddceaedc5d6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2213cf9588c545f2b200186766ac425d",
            "placeholder": "​",
            "style": "IPY_MODEL_d56ccfc975d949b7a6cbe7b9a89a1101",
            "value": " 124/124 [00:00&lt;00:00, 2.25kB/s]"
          }
        },
        "8d825e3d51524720ada2f1ecd9a9ffd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a17934807c140eeb8c0a5cf80c4cb50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d2254be42234d24b044f6c9c0d49119": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9eddddc3c9034e6da5bf612da55f5714": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9548afdd0d154788ba3edb96c326c85f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2213cf9588c545f2b200186766ac425d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d56ccfc975d949b7a6cbe7b9a89a1101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ff3907fe32a14ae681a6a01933e6eaf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f08eac6bbad843ca95157d035c1ef043",
              "IPY_MODEL_7e8af1c44d984d99a2b27e80c05cec75",
              "IPY_MODEL_0a381955a75148d5a59815efb2b352bd"
            ],
            "layout": "IPY_MODEL_7ac42549074746b48926a9cffe0a542c"
          }
        },
        "f08eac6bbad843ca95157d035c1ef043": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_55b65d9fa1e44149bfc763a8c99e7c2a",
            "placeholder": "​",
            "style": "IPY_MODEL_cd7ca2bbf9b04de4842b316ea7a5535f",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "7e8af1c44d984d99a2b27e80c05cec75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c0b4f2c4ef14413bc24340afc040431",
            "max": 26,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6442fa46c934428998cf8e3d5835e8bc",
            "value": 26
          }
        },
        "0a381955a75148d5a59815efb2b352bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8da520e3f829425b83e9b92149f792ad",
            "placeholder": "​",
            "style": "IPY_MODEL_78c5836dbb6b4538abcc3cd348c5b958",
            "value": " 26.0/26.0 [00:00&lt;00:00, 404B/s]"
          }
        },
        "7ac42549074746b48926a9cffe0a542c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55b65d9fa1e44149bfc763a8c99e7c2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd7ca2bbf9b04de4842b316ea7a5535f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c0b4f2c4ef14413bc24340afc040431": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6442fa46c934428998cf8e3d5835e8bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8da520e3f829425b83e9b92149f792ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78c5836dbb6b4538abcc3cd348c5b958": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f9fa0a2a0614bae87f7c698d043a3b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6c5fb86539584d6dad2af92729b9a1ef",
              "IPY_MODEL_1eb460d0dad44857a14646782366c7c1",
              "IPY_MODEL_67c1873c51d34503833366af483744a3"
            ],
            "layout": "IPY_MODEL_713fef79f5d441b18f372c64d9569551"
          }
        },
        "6c5fb86539584d6dad2af92729b9a1ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b5d59490f264d03a9ac52897ae94d33",
            "placeholder": "​",
            "style": "IPY_MODEL_42519c4cca864ad78595214e86079c8f",
            "value": "vocab.json: 100%"
          }
        },
        "1eb460d0dad44857a14646782366c7c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd3444ba231347378ae1a7ffa4181648",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d701e46a5472481ea942b613e3506f03",
            "value": 1042301
          }
        },
        "67c1873c51d34503833366af483744a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2effa7f78034f5c98a2c7f2d12b7ec3",
            "placeholder": "​",
            "style": "IPY_MODEL_8ee1c3f4f84f49b9aa78590d20e03005",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 27.2MB/s]"
          }
        },
        "713fef79f5d441b18f372c64d9569551": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b5d59490f264d03a9ac52897ae94d33": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42519c4cca864ad78595214e86079c8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd3444ba231347378ae1a7ffa4181648": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d701e46a5472481ea942b613e3506f03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e2effa7f78034f5c98a2c7f2d12b7ec3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ee1c3f4f84f49b9aa78590d20e03005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "00d232fe28f74a1ea9fad0510586ff9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e2e8c12232854688802988193dd6bdc8",
              "IPY_MODEL_3129da774c2c4298afbf3d766ca7f428",
              "IPY_MODEL_0d36758d65ed404e8181bdc4e2a9d26b"
            ],
            "layout": "IPY_MODEL_3e75401bc3bf42da8f9e1a377401b69d"
          }
        },
        "e2e8c12232854688802988193dd6bdc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbd25c280a554d50ace7384d5939e335",
            "placeholder": "​",
            "style": "IPY_MODEL_3371add482984a3bab5c31379484af71",
            "value": "merges.txt: 100%"
          }
        },
        "3129da774c2c4298afbf3d766ca7f428": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_001363aab0a143a683a07e48cbbf5594",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c87375d637f74dc4ab4bab00852c758c",
            "value": 456318
          }
        },
        "0d36758d65ed404e8181bdc4e2a9d26b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5f27258ef4a4b9e8b7d13387b14d9a1",
            "placeholder": "​",
            "style": "IPY_MODEL_d201d648b932443ca9141aa9129e5661",
            "value": " 456k/456k [00:00&lt;00:00, 12.9MB/s]"
          }
        },
        "3e75401bc3bf42da8f9e1a377401b69d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbd25c280a554d50ace7384d5939e335": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3371add482984a3bab5c31379484af71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "001363aab0a143a683a07e48cbbf5594": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c87375d637f74dc4ab4bab00852c758c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b5f27258ef4a4b9e8b7d13387b14d9a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d201d648b932443ca9141aa9129e5661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a2f04862b3a4c36becf5949c67a14a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8f57a09202cf4cddb2ba697ecfc78daf",
              "IPY_MODEL_a9fae2f4af744d33bddeeb48420cda3e",
              "IPY_MODEL_2994daa8857945509c97e3f2cdb4ee97"
            ],
            "layout": "IPY_MODEL_593dfccfa5ca4e56a32e9cead70714d2"
          }
        },
        "8f57a09202cf4cddb2ba697ecfc78daf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f3d428e20334a3da4c82d0fc2e3eb3e",
            "placeholder": "​",
            "style": "IPY_MODEL_5e028ae3b1824c74a0d610129dcc8246",
            "value": "tokenizer.json: 100%"
          }
        },
        "a9fae2f4af744d33bddeeb48420cda3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28ba4c7a70294e589cb2ced5766072e4",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d3a9c7752afb49a5bb1ea70bdb8744fe",
            "value": 1355256
          }
        },
        "2994daa8857945509c97e3f2cdb4ee97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09eed5891aa647da9f7c9beceac50622",
            "placeholder": "​",
            "style": "IPY_MODEL_4868e95bf5ec4cecab53e5e65862aea6",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 13.1MB/s]"
          }
        },
        "593dfccfa5ca4e56a32e9cead70714d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f3d428e20334a3da4c82d0fc2e3eb3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e028ae3b1824c74a0d610129dcc8246": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28ba4c7a70294e589cb2ced5766072e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3a9c7752afb49a5bb1ea70bdb8744fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "09eed5891aa647da9f7c9beceac50622": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4868e95bf5ec4cecab53e5e65862aea6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/argonne-lcf/ai-science-training-series/blob/main/04_intro_to_llms/IntroLLMs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Introduction to Language models (LMs)\n",
        "\n",
        "Author: Archit Vasan , including materials on LLMs by Varuni Sastri and Carlo Graziani at Argonne, and discussion/editorial work by Taylor Childers, Bethany Lusch, and Venkat Vishwanath (Argonne)\n",
        "\n",
        "Inspiration from the blog posts \"The Illustrated Transformer\" and \"The Illustrated GPT2\" by Jay Alammar, highly recommended reading.\n",
        "\n",
        "Although the name \"language models\" is derived from Natural Language Processing, the models used in these approaches can be applied to diverse scientific applications as illustrated below."
      ],
      "metadata": {
        "id": "FU00jtHMWQa9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Outline\n",
        "During this session I will cover:\n",
        "1. Scientific applications for language models\n",
        "2. General overview of Transformers\n",
        "3. Tokenization\n",
        "4. Model Architecture\n",
        "5. Pipeline using HuggingFace  \n",
        "6. Model loading"
      ],
      "metadata": {
        "id": "kIMoNmPLWZUv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Modeling Sequential Data\n",
        "\n",
        "Sequences are variable-length lists with data in subsequent iterations that depends on previous iterations (or tokens).\n",
        "\n",
        "Mathematically:\n",
        "A sequence is a list of tokens: $$T = [t_1, t_2, t_3,...,t_N]$$ where each token within the list depends on the others with a particular probability:\n",
        "\n",
        "$$P(t_N | t_{N-1}, ..., t_3, t_2, t_1)$$"
      ],
      "metadata": {
        "id": "mXf7L02LXrzk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The purpose of sequential modeling is to learn these probabilities for possible tokens in a distribution to perform various tasks including:\n",
        "* Sequence generation based on a prompt\n",
        "* Language translation (e.g. English --> French)\n",
        "* Property prediction (predicting a property based on an entire sequence)\n",
        "* Identifying mistakes or missing elements in sequential data"
      ],
      "metadata": {
        "id": "dpeNk1umXs9r"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Scientific sequential data modeling examples"
      ],
      "metadata": {
        "id": "gY7pqmOiX0Sd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        " ### Nucleic acid sequences + genomic data\n",
        "\n",
        " <div style=\"text-align: center\">\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/RNA-codons.svg.png?raw=1\"\n",
        " width=\"200\">\n",
        "</div>\n",
        "\n",
        "Nucleic acid sequences can be used to predict translation of proteins, mutations, and gene expression levels.\n"
      ],
      "metadata": {
        "id": "zyUUYza0X4WV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here is an image of GenSLM. This is a language model developed by Argonne researchers that can model genomic information in a single model. It was shown to model the evolution of SARS-COV2 without expensive experiments.\n",
        "\n",
        "<div>\n",
        "\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/genslm.png?raw=1\" width=\"450\"/>\n",
        "</div>\n",
        "\n",
        "[Zvyagin et. al 2022. BioRXiv](https://www.biorxiv.org/content/10.1101/2022.10.10.511571v1)"
      ],
      "metadata": {
        "id": "3w46oJ-EZCLk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Protein sequences\n",
        "Protein sequences can be used to predict folding structure, protein-protein interactions, chemical/binding properties, protein function and many more properties.\n",
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/Protein-Structure-06.png?raw=1\" width=\"400\"/>\n",
        "</div>\n",
        "\n",
        "<div>\n",
        "<img src=\"https://github.com/argonne-lcf/ai-science-training-series/blob/main/04_intro_to_llms/images/ESMFold.png?raw=1\" width=\"700\"/>\n",
        "</div>\n",
        "\n",
        "[Lin et. al. 2023. Science](https://www.science.org/doi/10.1126/science.ade2574)"
      ],
      "metadata": {
        "id": "f8cvujrcZIrp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Other applications:\n",
        "\n",
        "* Biomedical text\n",
        "* SMILES strings\n",
        "* Weather predictions\n",
        "* Interfacing with simulations such as molecular dynamics simulation"
      ],
      "metadata": {
        "id": "2rmNM-_4ZQYG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Overview of Language models\n",
        "\n",
        "We will now briefly talk about the progression of language models.\n",
        "\n",
        "### Transformers\n",
        "\n",
        "The most common LMs base their design on the Transformer architecture that was introduced in 2017 in the \"Attention is all you need\" paper.\n",
        "\n",
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/attention_is_all_you_need.png?raw=1\" width=\"500\"/>\n",
        "</div>\n",
        "\n",
        "[Vaswani 2017. Advances in Neural Information Processing Systems](https://arxiv.org/pdf/1706.03762)\n",
        "\n",
        "Since then a multitude of LLM architectures have been designed."
      ],
      "metadata": {
        "id": "KRwwYv5KckDx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/en_chapter1_transformers_chrono.svg?raw=1\" width=\"600\"/>\n",
        "</div>\n",
        "\n",
        "[HuggingFace NLP Course](https://huggingface.co/learn/nlp-course/chapter1/4)"
      ],
      "metadata": {
        "id": "N7EVxmOlcy3e"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Coding example of LLMs in action!\n",
        "\n",
        "Let's look at an example of running inference with a LLM as a block box to generate text given a prompt and we will also initiate a training loop for an LLM:\n",
        "\n",
        "Here, we will use the `transformers` library which is as part of HuggingFace, a repository of different models, tokenizers and information on how to apply these models"
      ],
      "metadata": {
        "id": "ENHDic3kdym_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "*Warning: Large Language Models are only as good as their training data. They have no ethics, no judgement, or editing ability. We will be using some pretrained models from Hugging Face which used wide samples of internet hosted text. The datasets have not been strictly filtered to restrict all malign content so the generated text may be surprisingly dark or questionable. They do not reflect our core values and are only used for demonstration purposes.*"
      ],
      "metadata": {
        "id": "6PKd2oA4d4jz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "Uncomment below section if running on sophia jupyter notebook\n",
        "'''\n",
        "# import os\n",
        "# os.environ[\"HTTP_PROXY\"]=\"proxy.alcf.anl.gov:3128\"\n",
        "# os.environ[\"HTTPS_PROXY\"]=\"proxy.alcf.anl.gov:3128\"\n",
        "# os.environ[\"http_proxy\"]=\"proxy.alcf.anl.gov:3128\"\n",
        "# os.environ[\"https_proxy\"]=\"proxy.alcf.anl.gov:3128\"\n",
        "# os.environ[\"ftp_proxy\"]=\"proxy.alcf.anl.gov:3128\""
      ],
      "metadata": {
        "id": "QRnnmrk9M4ds",
        "outputId": "b6551823-7c2a-4ddb-cd38-5ea5700f8fea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nUncomment below section if running on sophia jupyter notebook\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install pandas\n",
        "!pip install torch"
      ],
      "metadata": {
        "id": "iStN3PrgWQ5t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "981f2945-1c24-4edd-e95d-90483c518319"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.6.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.2.2)\n",
            "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas) (1.26.4)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.4.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.6.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer,AutoModelForCausalLM, AutoConfig\n",
        "input_text = \"My dog really wanted to eat icecream because\"\n",
        "from transformers import pipeline\n",
        "generator = pipeline(\"text-generation\", model=\"gpt2\")\n",
        "pipe = pipeline(\"text-generation\", model=\"gpt2\")\n",
        "generator(input_text, max_length=20, num_return_sequences=5)"
      ],
      "metadata": {
        "id": "V_BCzCJUMngf",
        "outputId": "7d563083-5a2e-4660-cef8-9e6e0b249026",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': 'My dog really wanted to eat icecream because the ice cream was really, really tasty,\" Raine'},\n",
              " {'generated_text': 'My dog really wanted to eat icecream because it was so bad.\"\\n\\nIt was about the'},\n",
              " {'generated_text': 'My dog really wanted to eat icecream because they wanted to use it as a base for the ice'},\n",
              " {'generated_text': 'My dog really wanted to eat icecream because the owner could get it in a store if he needed'},\n",
              " {'generated_text': \"My dog really wanted to eat icecream because she didn't think we could even get her to it\"}]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## What's going on under the hood?\n",
        "There are two components that are \"black-boxes\" here:\n",
        "\n",
        "1. The method for tokenization\n",
        "2. The model that generates novel text.\n"
      ],
      "metadata": {
        "id": "HAQ55_VxeWCD"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenization and embedding of sequential data"
      ],
      "metadata": {
        "id": "0P9TT1O50jNA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Humans can inherently understand language data because they previously learned phonetic sounds.\n",
        "\n",
        "Machines don’t have phonetic knowledge so they need to be told how to break text into standard units to process it.\n",
        "\n",
        "They use a system called “tokenization”, where sequences of text are broken into smaller parts, or “tokens”, and then fed as input.\n",
        "\n",
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/text-processing---machines-vs-humans.png?raw=1\" width=\"400\"/>\n",
        "</div>"
      ],
      "metadata": {
        "id": "pFafk6sS0oa2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenization is a data preprocessing step which transforms the raw text data into a format suitable for machine learning models. Tokenizers break down raw text into smaller units called tokens. These tokens are what is fed into the language models. Based on the type and configuration of the tokenizer, these tokens can be words, subwords, or characters."
      ],
      "metadata": {
        "id": "EjicllKr1PZR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Types of tokenizers:\n",
        "\n",
        "1. Character Tokenizers: Split text into individual characters.\n",
        "2. Word Tokenizers: Split text into words based on whitespace or punctuation.\n",
        "3. Subword Tokenizers: Split text into subword units, such as morphemes or character n-grams. Common subword tokenization algorithms include:\n",
        "  1. Byte-Pair Encoding (BPE),\n",
        "  2. SentencePiece,\n",
        "  3. WordPiece.\n",
        "\n",
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/tokenization_image.webp?raw=1\" width=\"400\"/>\n",
        "</div>\n",
        "\n",
        "[nlpiation](https://nlpiation.medium.com/how-to-use-huggingfaces-transformers-pre-trained-tokenizers-e029e8d6d1fa)"
      ],
      "metadata": {
        "id": "odlW24TP1dm4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example of tokenization\n",
        "Let's look at an example of tokenization using byte-pair encoding."
      ],
      "metadata": {
        "id": "rN3zt8XBbWfw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "# A utility function to tokenize a sequence and print out some information about it.\n",
        "\n",
        "def tokenization_summary(tokenizer, sequence):\n",
        "\n",
        "    # get the vocabulary\n",
        "    vocab = tokenizer.vocab\n",
        "    # Number of entries to print\n",
        "    n = 10\n",
        "\n",
        "    # Print subset of the vocabulary\n",
        "    print(\"Subset of tokenizer.vocab:\")\n",
        "    for i, (token, index) in enumerate(tokenizer.vocab.items()):\n",
        "        print(f\"{token}: {index}\")\n",
        "        if i >= n - 1:\n",
        "            break\n",
        "\n",
        "    print(\"Vocab size of the tokenizer = \", len(vocab))\n",
        "    print(\"------------------------------------------\")\n",
        "\n",
        "    # .tokenize chunks the existing sequence into different tokens based on the rules and vocab of the tokenizer.\n",
        "    tokens = tokenizer.tokenize(sequence)\n",
        "    print(\"Tokens : \", tokens)\n",
        "    print(\"------------------------------------------\")\n",
        "\n",
        "    # .convert_tokens_to_ids or .encode or .tokenize converts the tokens to their corresponding numerical representation.\n",
        "    #  .convert_tokens_to_ids has a 1-1 mapping between tokens and numerical representation\n",
        "    # ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "    # print(\"encoded Ids: \", ids)\n",
        "\n",
        "    # .encode also adds additional information like Start of sequence tokens and End of sequene\n",
        "    print(\"tokenized sequence : \", tokenizer.encode(sequence))\n",
        "\n",
        "    # .tokenizer has additional information about attention_mask.\n",
        "    # encode = tokenizer(sequence)\n",
        "    # print(\"Encode sequence : \", encode)\n",
        "    # print(\"------------------------------------------\")\n",
        "\n",
        "    # .decode decodes the ids to raw text\n",
        "    ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "    decode = tokenizer.decode(ids)\n",
        "    print(\"Decode sequence : \", decode)\n",
        "\n",
        "\n",
        "tokenizer_1  =  AutoTokenizer.from_pretrained(\"gpt2\") # GPT-2 uses \"Byte-Pair Encoding (BPE)\"\n",
        "\n",
        "sequence = \"Counselor, please adjust your Zoom filter to appear as a human, rather than as a cat\"\n",
        "\n",
        "tokenization_summary(tokenizer_1, sequence)"
      ],
      "metadata": {
        "id": "BKcviucieKOx",
        "outputId": "376bdcd8-16fd-4a2d-c436-a8cfa29d1788",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Subset of tokenizer.vocab:\n",
            "arning: 4228\n",
            "ATE: 6158\n",
            "GS: 14313\n",
            "foo: 21943\n",
            "Ġghosts: 25899\n",
            "Ġamino: 23206\n",
            "ĠDonation: 41654\n",
            "anas: 15991\n",
            "ions: 507\n",
            "ĠVID: 32247\n",
            "Vocab size of the tokenizer =  50257\n",
            "------------------------------------------\n",
            "Tokens :  ['Coun', 'sel', 'or', ',', 'Ġplease', 'Ġadjust', 'Ġyour', 'ĠZoom', 'Ġfilter', 'Ġto', 'Ġappear', 'Ġas', 'Ġa', 'Ġhuman', ',', 'Ġrather', 'Ġthan', 'Ġas', 'Ġa', 'Ġcat']\n",
            "------------------------------------------\n",
            "tokenized sequence :  [31053, 741, 273, 11, 3387, 4532, 534, 40305, 8106, 284, 1656, 355, 257, 1692, 11, 2138, 621, 355, 257, 3797]\n",
            "Decode sequence :  Counselor, please adjust your Zoom filter to appear as a human, rather than as a cat\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Token embedding:\n",
        "\n",
        "Words are turned into vectors based on their location within a vocabulary."
      ],
      "metadata": {
        "id": "qARCA2NjcENE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The strategy of choice for learning language structure from tokenized text is to find a clever way to map each token into a moderate-dimension vector space, adjusting the mapping so that\n",
        "\n",
        "Similar, or associated tokens take up residence nearby each other, and different regions of the space correspond to different position in the sequence.\n",
        "Such a mapping from token ID to a point in a vector space is called a token embedding. The dimension of the vector space is often high (e.g. 1024-dimensional), but much smaller than the vocabulary size (30,000--500,000).\n",
        "\n",
        "Various approaches have been attempted for generating such embeddings, including static algorithms that operate on a corpus of tokenized data as preprocessors for NLP tasks. Transformers, however, adjust their embeddings during training."
      ],
      "metadata": {
        "id": "eoyvSYNicI-r"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Transformer Model Architecture"
      ],
      "metadata": {
        "id": "47Rpe74mg4Ji"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's look at the base elements that\n",
        "make up a Transformer by dissecting the popular GPT2 model"
      ],
      "metadata": {
        "id": "1CY2gHjfhADT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "print(model)"
      ],
      "metadata": {
        "id": "grSmeSf6cEsa",
        "outputId": "13ca18c8-e35e-4819-ec66-a0269e673424",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPT2LMHeadModel(\n",
            "  (transformer): GPT2Model(\n",
            "    (wte): Embedding(50257, 768)\n",
            "    (wpe): Embedding(1024, 768)\n",
            "    (drop): Dropout(p=0.1, inplace=False)\n",
            "    (h): ModuleList(\n",
            "      (0-11): 12 x GPT2Block(\n",
            "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
            "        (attn): GPT2SdpaAttention(\n",
            "          (c_attn): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
            "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
            "        (mlp): GPT2MLP(\n",
            "          (c_fc): Conv1D()\n",
            "          (c_proj): Conv1D()\n",
            "          (act): NewGELUActivation()\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "GPT2 is an example of a Transformer Decoder which is used to generate novel text.\n",
        "\n",
        "Decoder models use only the decoder of a Transformer model. At each stage, for a given word the attention layers can only access the words positioned before it in the sentence. These models are often called auto-regressive models. The pretraining of decoder models usually revolves around predicting the next word in the sentence.\n",
        "\n",
        "These models are best suited for tasks involving text generation."
      ],
      "metadata": {
        "id": "58nP_GM9nCAG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The architecture of GPT-2 is inspired by the paper: \"Generating Wikipedia by Summarizing Long Sequences\" which is another arrangement of the transformer block that can do language modeling. This model threw away the encoder and thus is known as the “Transformer-Decoder”."
      ],
      "metadata": {
        "id": "nDtN24y-nRX4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/transformer-decoder-intro.png?raw=1\" width=\"500\"/>\n",
        "</div>"
      ],
      "metadata": {
        "id": "BhDtVlwBnU9U"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "[Illustrated GPT2](https://jalammar.github.io/illustrated-gpt2/)"
      ],
      "metadata": {
        "id": "Oo2WJBDCoB_1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Key components of the transformer architecture include:\n",
        "\n",
        "* Input Embeddings: Word embedding or word vectors help us represent words or text as a numeric vector where words with similar meanings have the similar representation.\n",
        "\n",
        "* Positional Encoding: Injects information about the position of words in a sequence, helping the model understand word order.\n",
        "\n",
        "* Self-Attention Mechanism: Allows the model to weigh the importance of different words in a sentence, enabling it to effectively capture contextual information.\n",
        "\n",
        "* Feedforward Neural Networks: Process information from self-attention layers to generate output for each word/token.\n",
        "\n",
        "* Layer Normalization and Residual Connections: Aid in stabilizing training and mitigating the vanishing gradient problem.\n",
        "\n",
        "* Transformer Blocks: Comprised of multiple layers of self-attention and feedforward neural networks, stacked together to form the model."
      ],
      "metadata": {
        "id": "ftwZNTlgorbe"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Attention mechanisms\n",
        "\n",
        "Since attention mechanisms are arguably the most powerful component of the Transformer, let's discuss this in a little more detail.\n",
        "\n",
        "Suppose the following sentence is an input sentence we want to translate using an LLM:\n",
        "\n",
        "`”The animal didn't cross the street because it was too tired”`\n",
        "\n",
        "To understand a full sentence, the model needs to understand what each word means in relation to other words.\n",
        "\n",
        "For example, when we read the sentence:\n",
        "`”The animal didn't cross the street because it was too tired”`\n",
        "we know intuitively that the word `\"it\"` refers to `\"animal\"`, the state for `\"it\"` is `\"tired\"`, and the associated action is `\"didn't cross\"`.\n",
        "\n",
        "However, the model needs a way to learn all of this information in a simple yet generalizable way.\n",
        "What makes Transformers particularly powerful compared to earlier sequential architectures is how it encodes context with the **self-attention mechanism**.\n",
        "\n",
        "As the model processes each word in the input sequence, attention looks at other positions in the input sequence for clues to a better understanding for this word."
      ],
      "metadata": {
        "id": "oCRILpZAo-lx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div>\n",
        "<img src=\"https://github.com/architvasan/ai_science_local/blob/main/images/transformer_self-attention_visualization.png?raw=1\" width=\"400\"/>\n",
        "</div>\n",
        "\n",
        "[The Illustrated Transformer](https://jalammar.github.io/illustrated-transformer/)"
      ],
      "metadata": {
        "id": "wbHnj21AxfPc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Multi-head attention\n",
        "In practice, multiple attention heads are used simultaneously.\n",
        "\n",
        "This:\n",
        "* Expands the model’s ability to focus on different positions.\n",
        "* Prevents the attention to be dominated by the word itself."
      ],
      "metadata": {
        "id": "UDuyVk8kSkJ5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Let's see multi-head attention mechanisms in action!"
      ],
      "metadata": {
        "id": "ouKwvZg5yKof"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We are going to use the powerful visualization tool bertviz, which allows an interactive experience of the attention mechanisms. Normally these mechanisms are abstracted away but this will allow us to inspect our model in more detail."
      ],
      "metadata": {
        "id": "5UAbwlrvyQzu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install bertviz"
      ],
      "metadata": {
        "id": "NlVH1kBpnDH5",
        "outputId": "2bb130fb-8a42-49f3-e7af-10548256a6ee",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting bertviz\n",
            "  Downloading bertviz-1.4.0-py3-none-any.whl.metadata (19 kB)\n",
            "Requirement already satisfied: transformers>=2.0 in /usr/local/lib/python3.10/dist-packages (from bertviz) (4.44.2)\n",
            "Requirement already satisfied: torch>=1.0 in /usr/local/lib/python3.10/dist-packages (from bertviz) (2.4.1+cu121)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from bertviz) (4.66.5)\n",
            "Collecting boto3 (from bertviz)\n",
            "  Downloading boto3-1.35.44-py3-none-any.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bertviz) (2.32.3)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from bertviz) (2024.9.11)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from bertviz) (0.2.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (3.16.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (1.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (3.4.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.0->bertviz) (2024.6.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (0.24.7)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (6.0.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers>=2.0->bertviz) (0.19.1)\n",
            "Collecting botocore<1.36.0,>=1.35.44 (from boto3->bertviz)\n",
            "  Downloading botocore-1.35.44-py3-none-any.whl.metadata (5.7 kB)\n",
            "Collecting jmespath<2.0.0,>=0.7.1 (from boto3->bertviz)\n",
            "  Downloading jmespath-1.0.1-py3-none-any.whl.metadata (7.6 kB)\n",
            "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3->bertviz)\n",
            "  Downloading s3transfer-0.10.3-py3-none-any.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->bertviz) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bertviz) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bertviz) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bertviz) (2024.8.30)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.36.0,>=1.35.44->boto3->bertviz) (2.8.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0->bertviz) (3.0.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.0->bertviz) (1.3.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.36.0,>=1.35.44->boto3->bertviz) (1.16.0)\n",
            "Downloading bertviz-1.4.0-py3-none-any.whl (157 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.6/157.6 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading boto3-1.35.44-py3-none-any.whl (139 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading botocore-1.35.44-py3-none-any.whl (12.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.6/12.6 MB\u001b[0m \u001b[31m48.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
            "Downloading s3transfer-0.10.3-py3-none-any.whl (82 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.6/82.6 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: jmespath, botocore, s3transfer, boto3, bertviz\n",
            "Successfully installed bertviz-1.4.0 boto3-1.35.44 botocore-1.35.44 jmespath-1.0.1 s3transfer-0.10.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's load in the model, GPT2 and look at the attention mechanisms.\n",
        "\n",
        "**Hint... click on the different blocks in the visualization to see the attention**"
      ],
      "metadata": {
        "id": "xQmakRo8yc26"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModel, utils, AutoModelForCausalLM\n",
        "\n",
        "from bertviz import model_view\n",
        "utils.logging.set_verbosity_error()  # Suppress standard warnings\n",
        "\n",
        "model_name = 'openai-community/gpt2'\n",
        "input_text = \"No, I am your father\"\n",
        "model = AutoModelForCausalLM.from_pretrained(model_name, output_attentions=True)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "inputs = tokenizer.encode(input_text, return_tensors='pt')  # Tokenize input text\n",
        "outputs = model(inputs)  # Run model\n",
        "attention = outputs[-1]  # Retrieve attention from model outputs\n",
        "tokens = tokenizer.convert_ids_to_tokens(inputs[0])  # Convert input ids to token strings\n",
        "model_view(attention, tokens)  # Display model view"
      ],
      "metadata": {
        "id": "gwlwH1E2yZQt",
        "outputId": "4b6dabb4-b5d7-487b-9d9d-9f8cc9817c7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 907,
          "referenced_widgets": [
            "ad748b4c592645cb90d427c9012d7368",
            "a34cdc4592cf46ad9ccf69caa52363bd",
            "c1eac1886ea84c87bed161c960569d3d",
            "ca50498e237843709d8c44e7ea0e94b5",
            "8f4387f0bb3e402ab1ab4d443c693f96",
            "47316e9804804c8d9c9b031fa3d86052",
            "6cc5bd956c694bc5b63c0c7bba2a29f8",
            "93728bade1714bcabbe90f1b9b95b7c2",
            "83cea646339b4e74a9582827afdf5753",
            "c765d253aa974bc59654f0398afb964d",
            "3d71478078bc4743a580dc6200cccbd0",
            "099e1574f4a14c308be2364d0bba2a76",
            "6f822f4ae7734ce09736d50bf17ad99e",
            "93fbd0ce38df488fa725d7c584344e6b",
            "ee56c1c4c2be4086883a95e15adc655d",
            "846bb66c196540febf6a7bc32b60dc3a",
            "37dcd77ea3154208be44bfa35a88fa90",
            "fdd9ba7947794f4aa0e2782e3b379f61",
            "64fa9056ffef4ab3a4e2db085a48cbfb",
            "acee154ece2e4a939f42236ec38de871",
            "46acb20f74174029962ecb8142c576fb",
            "b58662cfa5fb4406a171135e01d25368",
            "c770f4a4cad0435ebedbb932c1431f52",
            "14dab4fee3d34783bf838cfbd4bdbcaa",
            "d00cd656bc4f4f0d9cf52b942fceb155",
            "d5fbc43ad5204bb88b11ddceaedc5d6d",
            "8d825e3d51524720ada2f1ecd9a9ffd5",
            "9a17934807c140eeb8c0a5cf80c4cb50",
            "7d2254be42234d24b044f6c9c0d49119",
            "9eddddc3c9034e6da5bf612da55f5714",
            "9548afdd0d154788ba3edb96c326c85f",
            "2213cf9588c545f2b200186766ac425d",
            "d56ccfc975d949b7a6cbe7b9a89a1101",
            "ff3907fe32a14ae681a6a01933e6eaf5",
            "f08eac6bbad843ca95157d035c1ef043",
            "7e8af1c44d984d99a2b27e80c05cec75",
            "0a381955a75148d5a59815efb2b352bd",
            "7ac42549074746b48926a9cffe0a542c",
            "55b65d9fa1e44149bfc763a8c99e7c2a",
            "cd7ca2bbf9b04de4842b316ea7a5535f",
            "0c0b4f2c4ef14413bc24340afc040431",
            "6442fa46c934428998cf8e3d5835e8bc",
            "8da520e3f829425b83e9b92149f792ad",
            "78c5836dbb6b4538abcc3cd348c5b958",
            "3f9fa0a2a0614bae87f7c698d043a3b5",
            "6c5fb86539584d6dad2af92729b9a1ef",
            "1eb460d0dad44857a14646782366c7c1",
            "67c1873c51d34503833366af483744a3",
            "713fef79f5d441b18f372c64d9569551",
            "4b5d59490f264d03a9ac52897ae94d33",
            "42519c4cca864ad78595214e86079c8f",
            "bd3444ba231347378ae1a7ffa4181648",
            "d701e46a5472481ea942b613e3506f03",
            "e2effa7f78034f5c98a2c7f2d12b7ec3",
            "8ee1c3f4f84f49b9aa78590d20e03005",
            "00d232fe28f74a1ea9fad0510586ff9c",
            "e2e8c12232854688802988193dd6bdc8",
            "3129da774c2c4298afbf3d766ca7f428",
            "0d36758d65ed404e8181bdc4e2a9d26b",
            "3e75401bc3bf42da8f9e1a377401b69d",
            "fbd25c280a554d50ace7384d5939e335",
            "3371add482984a3bab5c31379484af71",
            "001363aab0a143a683a07e48cbbf5594",
            "c87375d637f74dc4ab4bab00852c758c",
            "b5f27258ef4a4b9e8b7d13387b14d9a1",
            "d201d648b932443ca9141aa9129e5661",
            "0a2f04862b3a4c36becf5949c67a14a5",
            "8f57a09202cf4cddb2ba697ecfc78daf",
            "a9fae2f4af744d33bddeeb48420cda3e",
            "2994daa8857945509c97e3f2cdb4ee97",
            "593dfccfa5ca4e56a32e9cead70714d2",
            "5f3d428e20334a3da4c82d0fc2e3eb3e",
            "5e028ae3b1824c74a0d610129dcc8246",
            "28ba4c7a70294e589cb2ced5766072e4",
            "d3a9c7752afb49a5bb1ea70bdb8744fe",
            "09eed5891aa647da9f7c9beceac50622",
            "4868e95bf5ec4cecab53e5e65862aea6"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ad748b4c592645cb90d427c9012d7368"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "099e1574f4a14c308be2364d0bba2a76"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c770f4a4cad0435ebedbb932c1431f52"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ff3907fe32a14ae681a6a01933e6eaf5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3f9fa0a2a0614bae87f7c698d043a3b5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "00d232fe28f74a1ea9fad0510586ff9c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0a2f04862b3a4c36becf5949c67a14a5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js\"></script>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "      \n",
              "        <div id=\"bertviz-2fa5f9f15ab7437992290934271cfac7\" style=\"font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;\">\n",
              "            <span style=\"user-select:none\">\n",
              "                \n",
              "            </span>\n",
              "            <div id='vis'></div>\n",
              "        </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "/**\n",
              " * @fileoverview Transformer Visualization D3 javascript code.\n",
              " *\n",
              " * Based on: https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/visualization/attention.js\n",
              " *\n",
              " * Change log:\n",
              " *\n",
              " * 02/01/19  Jesse Vig   Initial implementation\n",
              " * 12/31/20  Jesse Vig   Support multiple visualizations in single notebook.\n",
              " * 01/19/21  Jesse Vig   Support light/dark modes\n",
              " * 02/06/21  Jesse Vig   Move require config from separate jupyter notebook step\n",
              " * 05/03/21  Jesse Vig   Adjust visualization height dynamically\n",
              " * 03/23/22  Daniel SC   Update requirement URLs for d3 and jQuery (source of bug not allowing end result to be displayed on browsers)\n",
              " **/\n",
              "\n",
              "require.config({\n",
              "  paths: {\n",
              "      d3: 'https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min',\n",
              "    jquery: 'https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.0/jquery.min',\n",
              "  }\n",
              "});\n",
              "\n",
              "requirejs(['jquery', 'd3'], function($, d3) {\n",
              "\n",
              "        const params = {\"attention\": [{\"name\": null, \"attn\": [[[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.961219847202301, 0.038780149072408676, 0.0, 0.0, 0.0, 0.0], [0.7466980218887329, 0.11987308412790298, 0.1334288865327835, 0.0, 0.0, 0.0], [0.5885030031204224, 0.13792069256305695, 0.21213732659816742, 0.06143900007009506, 0.0, 0.0], [0.657085657119751, 0.08996297419071198, 0.1275128275156021, 0.08361561596393585, 0.04182284325361252, 0.0], [0.272887259721756, 0.11203360557556152, 0.16639851033687592, 0.08467116951942444, 0.16952727735042572, 0.19448219239711761]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010616553947329521, 0.9893833994865417, 0.0, 0.0, 0.0, 0.0], [0.002467755926772952, 0.008448011241853237, 0.9890841841697693, 0.0, 0.0, 0.0], [0.00012328448065090925, 0.0018733128672465682, 0.013126945123076439, 0.9848763942718506, 0.0, 0.0], [0.0010669530602172017, 0.0011366248363628983, 0.0030349865555763245, 0.00157350511290133, 0.9931879043579102, 0.0], [0.00019792001694440842, 0.0010528122074902058, 0.0015437351539731026, 0.0009642756194807589, 3.492440009722486e-05, 0.9962062835693359]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4757843017578125, 0.5242156982421875, 0.0, 0.0, 0.0, 0.0], [0.5906046032905579, 0.2486610859632492, 0.1607343554496765, 0.0, 0.0, 0.0], [0.5529289841651917, 0.18856702744960785, 0.14457568526268005, 0.11392833292484283, 0.0, 0.0], [0.4509407877922058, 0.16486789286136627, 0.17318037152290344, 0.11748013645410538, 0.0935308039188385, 0.0], [0.42572465538978577, 0.17328648269176483, 0.15651948750019073, 0.07022645324468613, 0.0808701142668724, 0.09337276965379715]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6133622527122498, 0.38663774728775024, 0.0, 0.0, 0.0, 0.0], [0.06098503991961479, 0.03253461420536041, 0.9064803123474121, 0.0, 0.0, 0.0], [0.0067170835100114346, 0.0004012887948192656, 0.7572956085205078, 0.2355860024690628, 0.0, 0.0], [0.03722767531871796, 0.002948855748400092, 0.10081084072589874, 0.04142269492149353, 0.8175899982452393, 0.0], [0.04989771544933319, 0.00030758281354792416, 0.0024198247119784355, 0.0034334948286414146, 0.0006823905860073864, 0.9432590007781982]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9489555954933167, 0.05104443058371544, 0.0, 0.0, 0.0, 0.0], [0.6821408867835999, 0.13952411711215973, 0.17833493649959564, 0.0, 0.0, 0.0], [0.20366327464580536, 0.056414876133203506, 0.06399297714233398, 0.6759288907051086, 0.0, 0.0], [0.34195470809936523, 0.067254438996315, 0.0792618840932846, 0.17836201190948486, 0.3331669569015503, 0.0], [0.09464021027088165, 0.007428212556988001, 0.006983972620218992, 0.007184368558228016, 0.018724262714385986, 0.8650389313697815]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33834612369537354, 0.6616538763046265, 0.0, 0.0, 0.0, 0.0], [0.07855986058712006, 0.006165443453937769, 0.9152746796607971, 0.0, 0.0, 0.0], [0.016775960102677345, 0.00040376983815804124, 0.0033404543064534664, 0.9794798493385315, 0.0, 0.0], [0.02760043926537037, 0.0004441527707967907, 0.0006541680195368826, 0.0002266185765620321, 0.971074640750885, 0.0], [0.010248193517327309, 3.70155721611809e-05, 0.00016064071678556502, 2.7341899112798274e-05, 1.0187313819187693e-05, 0.98951655626297]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9825032949447632, 0.01749667525291443, 0.0, 0.0, 0.0, 0.0], [0.8874197006225586, 0.05467936769127846, 0.05790088325738907, 0.0, 0.0, 0.0], [0.6849910020828247, 0.12280681729316711, 0.04972028359770775, 0.14248184859752655, 0.0, 0.0], [0.6015853881835938, 0.09881892055273056, 0.07070116698741913, 0.1665254533290863, 0.062369052320718765, 0.0], [0.32325053215026855, 0.12567415833473206, 0.04432179033756256, 0.07076980173587799, 0.06606654077768326, 0.3699170649051666]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9191646575927734, 0.08083534240722656, 0.0, 0.0, 0.0, 0.0], [0.4598640501499176, 0.39703115820884705, 0.14310480654239655, 0.0, 0.0, 0.0], [0.3003872334957123, 0.22181734442710876, 0.3816152513027191, 0.09618015587329865, 0.0, 0.0], [0.18963924050331116, 0.13763725757598877, 0.20173496007919312, 0.2363216131925583, 0.2346668690443039, 0.0], [0.15410453081130981, 0.09489504247903824, 0.11902561038732529, 0.1027795746922493, 0.4317219853401184, 0.09747327864170074]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.364999920129776, 0.6350001096725464, 0.0, 0.0, 0.0, 0.0], [0.24595212936401367, 0.5519201755523682, 0.20212772488594055, 0.0, 0.0, 0.0], [0.27213579416275024, 0.40738630294799805, 0.251862108707428, 0.06861570477485657, 0.0, 0.0], [0.1024254709482193, 0.16683608293533325, 0.5248051881790161, 0.05445462465286255, 0.15147872269153595, 0.0], [0.2502949833869934, 0.22198131680488586, 0.1889997124671936, 0.10677109658718109, 0.1303267925977707, 0.10162606090307236]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6990506649017334, 0.300949364900589, 0.0, 0.0, 0.0, 0.0], [0.5107942819595337, 0.294864296913147, 0.1943414956331253, 0.0, 0.0, 0.0], [0.46047067642211914, 0.28051918745040894, 0.19174796342849731, 0.06726215779781342, 0.0, 0.0], [0.3764842748641968, 0.21120664477348328, 0.20214536786079407, 0.10207021236419678, 0.10809348523616791, 0.0], [0.30138450860977173, 0.20456179976463318, 0.1825033277273178, 0.11019382625818253, 0.16291265189647675, 0.03844384104013443]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.713158369064331, 0.28684163093566895, 0.0, 0.0, 0.0, 0.0], [0.4058799147605896, 0.18063293397426605, 0.41348710656166077, 0.0, 0.0, 0.0], [0.26554614305496216, 0.16985861957073212, 0.3358593285083771, 0.22873590886592865, 0.0, 0.0], [0.31385400891304016, 0.1831670105457306, 0.14928355813026428, 0.05377669632434845, 0.2999187409877777, 0.0], [0.20466554164886475, 0.18731112778186798, 0.15959149599075317, 0.06381770968437195, 0.03642303869128227, 0.3481910526752472]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6586242318153381, 0.34137576818466187, 0.0, 0.0, 0.0, 0.0], [0.5917777419090271, 0.3160034716129303, 0.09221877157688141, 0.0, 0.0, 0.0], [0.5477151870727539, 0.23586955666542053, 0.06145601347088814, 0.15495926141738892, 0.0, 0.0], [0.4587061107158661, 0.22439998388290405, 0.07887422293424606, 0.0992034301161766, 0.13881628215312958, 0.0], [0.32743728160858154, 0.19600822031497955, 0.06805708259344101, 0.0892510712146759, 0.11618075519800186, 0.20306557416915894]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9961552023887634, 0.0038448390550911427, 0.0, 0.0, 0.0, 0.0], [0.8594854474067688, 0.06906113028526306, 0.07145342975854874, 0.0, 0.0, 0.0], [0.3800053596496582, 0.04127567261457443, 0.5496612191200256, 0.029057808220386505, 0.0, 0.0], [0.21445205807685852, 0.050887417048215866, 0.43174412846565247, 0.25869303941726685, 0.04422337934374809, 0.0], [0.11175268143415451, 0.01759307272732258, 0.027507441118359566, 0.04086774215102196, 0.7754669785499573, 0.02681216411292553]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9285967946052551, 0.07140323519706726, 0.0, 0.0, 0.0, 0.0], [0.6077285408973694, 0.31214284896850586, 0.08012856543064117, 0.0, 0.0, 0.0], [0.49429091811180115, 0.2850370407104492, 0.11849313229322433, 0.1021789088845253, 0.0, 0.0], [0.41838788986206055, 0.2311791181564331, 0.08340615779161453, 0.11365945637226105, 0.1533673107624054, 0.0], [0.4221586287021637, 0.1291714906692505, 0.08740919828414917, 0.10163754224777222, 0.21230259537696838, 0.04732052609324455]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9786475896835327, 0.02135237492620945, 0.0, 0.0, 0.0, 0.0], [0.7749121189117432, 0.0651036873459816, 0.15998415648937225, 0.0, 0.0, 0.0], [0.6484923958778381, 0.07483129948377609, 0.14751604199409485, 0.12916021049022675, 0.0, 0.0], [0.5224636793136597, 0.06921812891960144, 0.13823404908180237, 0.11106594651937485, 0.15901818871498108, 0.0], [0.39645177125930786, 0.07325821369886398, 0.1293814778327942, 0.10642431676387787, 0.14864003658294678, 0.14584419131278992]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5525906085968018, 0.44740942120552063, 0.0, 0.0, 0.0, 0.0], [0.5585009455680847, 0.2176259309053421, 0.22387315332889557, 0.0, 0.0, 0.0], [0.5143128633499146, 0.15964668989181519, 0.1549196094274521, 0.1711207777261734, 0.0, 0.0], [0.5039963126182556, 0.11401881277561188, 0.11974020302295685, 0.12552586197853088, 0.13671880960464478, 0.0], [0.5061841607093811, 0.08567394316196442, 0.0890301838517189, 0.09759820997714996, 0.1027572825551033, 0.11875618249177933]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9242545366287231, 0.07574543356895447, 0.0, 0.0, 0.0, 0.0], [0.8257424235343933, 0.07932537794113159, 0.0949321910738945, 0.0, 0.0, 0.0], [0.7306379675865173, 0.08571833372116089, 0.08043932169675827, 0.1032044067978859, 0.0, 0.0], [0.6383240818977356, 0.07886383682489395, 0.07815023511648178, 0.08758097141981125, 0.117080919444561, 0.0], [0.5552157163619995, 0.07409115880727768, 0.06834889203310013, 0.07778602838516235, 0.09999320656061172, 0.12456496059894562]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8578913807868958, 0.14210855960845947, 0.0, 0.0, 0.0, 0.0], [0.6423036456108093, 0.16629023849964142, 0.19140605628490448, 0.0, 0.0, 0.0], [0.5530978441238403, 0.10609277337789536, 0.07821263372898102, 0.2625967860221863, 0.0, 0.0], [0.4012168049812317, 0.1222359910607338, 0.19347327947616577, 0.14164631068706512, 0.14142754673957825, 0.0], [0.40212583541870117, 0.1845075935125351, 0.07516805827617645, 0.058490391820669174, 0.14446339011192322, 0.13524475693702698]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9791558980941772, 0.020844046026468277, 0.0, 0.0, 0.0, 0.0], [0.8829841613769531, 0.062332455068826675, 0.05468333512544632, 0.0, 0.0, 0.0], [0.8105455040931702, 0.08617087453603745, 0.07321774214506149, 0.030065858736634254, 0.0, 0.0], [0.6819811463356018, 0.04990815743803978, 0.08296556025743484, 0.08369529247283936, 0.10144980996847153, 0.0], [0.4056687355041504, 0.07337656617164612, 0.08601398020982742, 0.06170930340886116, 0.13226443529129028, 0.24096696078777313]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9670192003250122, 0.03298087790608406, 0.0, 0.0, 0.0, 0.0], [0.8449065089225769, 0.08514498919248581, 0.06994851678609848, 0.0, 0.0, 0.0], [0.7123571038246155, 0.07896049320697784, 0.055410757660865784, 0.1532716304063797, 0.0, 0.0], [0.640261173248291, 0.0739755630493164, 0.044393062591552734, 0.14322136342525482, 0.09814884513616562, 0.0], [0.5073902606964111, 0.07523062080144882, 0.07754648476839066, 0.11362482607364655, 0.13947948813438416, 0.08672824501991272]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8487567901611328, 0.15124322474002838, 0.0, 0.0, 0.0, 0.0], [0.8415648341178894, 0.1210724338889122, 0.03736277297139168, 0.0, 0.0, 0.0], [0.7505517601966858, 0.11348951607942581, 0.061799556016922, 0.07415911555290222, 0.0, 0.0], [0.6614715456962585, 0.10242648422718048, 0.052934277802705765, 0.0752972960472107, 0.10787033289670944, 0.0], [0.6014202237129211, 0.11340402066707611, 0.05631927400827408, 0.07096727937459946, 0.10906262695789337, 0.04882659390568733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9445484280586243, 0.05545159429311752, 0.0, 0.0, 0.0, 0.0], [0.8874567747116089, 0.05474221706390381, 0.057801052927970886, 0.0, 0.0, 0.0], [0.8281887769699097, 0.06895007193088531, 0.05903473123908043, 0.04382641240954399, 0.0, 0.0], [0.6429893970489502, 0.06747540831565857, 0.11629702895879745, 0.05417948588728905, 0.11905863881111145, 0.0], [0.7367823123931885, 0.056119006127119064, 0.06857286393642426, 0.034219563007354736, 0.07875377684831619, 0.025552386417984962]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0002913392090704292, 0.9997085928916931, 0.0, 0.0, 0.0, 0.0], [0.0007981202797964215, 0.5288337469100952, 0.4703681766986847, 0.0, 0.0, 0.0], [0.0007648473256267607, 0.3451983630657196, 0.3085266649723053, 0.3455101251602173, 0.0, 0.0], [0.0010283150477334857, 0.24135911464691162, 0.23320142924785614, 0.2555714249610901, 0.26883965730667114, 0.0], [0.0009746808791533113, 0.17789694666862488, 0.1674315482378006, 0.18587610125541687, 0.18734434247016907, 0.2804763615131378]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8244927525520325, 0.17550718784332275, 0.0, 0.0, 0.0, 0.0], [0.12386851012706757, 0.044499825686216354, 0.8316316604614258, 0.0, 0.0, 0.0], [0.07924355566501617, 0.012965913861989975, 0.0015277150087058544, 0.9062628149986267, 0.0, 0.0], [0.08806417882442474, 0.02134108357131481, 0.0028886189684271812, 0.002845388138666749, 0.8848607540130615, 0.0], [0.09983226656913757, 0.03363394737243652, 0.005499998107552528, 0.00243305298499763, 0.0015082467580214143, 0.8570924997329712]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9646892547607422, 0.03531072288751602, 0.0, 0.0, 0.0, 0.0], [0.7529155015945435, 0.08733480423688889, 0.15974965691566467, 0.0, 0.0, 0.0], [0.42022842168807983, 0.09195113927125931, 0.23549841344356537, 0.25232207775115967, 0.0, 0.0], [0.3084895610809326, 0.05908143147826195, 0.3839128613471985, 0.15659146010875702, 0.09192463010549545, 0.0], [0.447904109954834, 0.04329312965273857, 0.07969177514314651, 0.1108192726969719, 0.2212459146976471, 0.09704578667879105]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.991096019744873, 0.008904017508029938, 0.0, 0.0, 0.0, 0.0], [0.9697675704956055, 0.026084477081894875, 0.004147926811128855, 0.0, 0.0, 0.0], [0.9082902073860168, 0.03320597484707832, 0.009421153925359249, 0.04908263683319092, 0.0, 0.0], [0.8949136137962341, 0.055445361882448196, 0.005577605217695236, 0.03150679171085358, 0.012556544505059719, 0.0], [0.8497737050056458, 0.028890114277601242, 0.0036648025270551443, 0.03751996532082558, 0.038428016006946564, 0.041723549365997314]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9984525442123413, 0.0015474484534934163, 0.0, 0.0, 0.0, 0.0], [0.4894773066043854, 0.48122045397758484, 0.02930225059390068, 0.0, 0.0, 0.0], [0.11772146075963974, 0.13121247291564941, 0.67023104429245, 0.08083496242761612, 0.0, 0.0], [0.13043726980686188, 0.0406869612634182, 0.2652043104171753, 0.41143471002578735, 0.15223680436611176, 0.0], [0.12661851942539215, 0.03275134041905403, 0.03567883372306824, 0.06039177253842354, 0.6021823287010193, 0.14237718284130096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9805176854133606, 0.019482314586639404, 0.0, 0.0, 0.0, 0.0], [0.7948848009109497, 0.12061930447816849, 0.0844959020614624, 0.0, 0.0, 0.0], [0.5612354874610901, 0.15743164718151093, 0.2033972144126892, 0.07793562114238739, 0.0, 0.0], [0.4258382320404053, 0.10742025077342987, 0.15123654901981354, 0.08754990994930267, 0.22795499861240387, 0.0], [0.24752630293369293, 0.024188317358493805, 0.030395260080695152, 0.08586960285902023, 0.5714336633682251, 0.04058684781193733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9887767434120178, 0.011223198845982552, 0.0, 0.0, 0.0, 0.0], [0.7572691440582275, 0.22317387163639069, 0.0195570457726717, 0.0, 0.0, 0.0], [0.5341880321502686, 0.22107607126235962, 0.17621813714504242, 0.0685177892446518, 0.0, 0.0], [0.17095306515693665, 0.08229431509971619, 0.5760217905044556, 0.11097602546215057, 0.0597548633813858, 0.0], [0.2487107813358307, 0.08880821615457535, 0.08980182558298111, 0.09729332476854324, 0.4413091838359833, 0.03407657518982887]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.84221351146698, 0.1577865481376648, 0.0, 0.0, 0.0, 0.0], [0.46841251850128174, 0.4610540270805359, 0.07053346186876297, 0.0, 0.0, 0.0], [0.25881412625312805, 0.46358928084373474, 0.18503478169441223, 0.09256181120872498, 0.0, 0.0], [0.18399572372436523, 0.29154250025749207, 0.17031069099903107, 0.2717295289039612, 0.08242153376340866, 0.0], [0.16469858586788177, 0.24727003276348114, 0.08770552277565002, 0.2257496416568756, 0.1774536818265915, 0.09712250530719757]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9919946193695068, 0.008005395531654358, 0.0, 0.0, 0.0, 0.0], [0.9068723320960999, 0.044065166264772415, 0.049062490463256836, 0.0, 0.0, 0.0], [0.8582221865653992, 0.055348340421915054, 0.040419381111860275, 0.046010058373212814, 0.0, 0.0], [0.7855251431465149, 0.041242457926273346, 0.08369290828704834, 0.04887617751955986, 0.04066329449415207, 0.0], [0.7856318354606628, 0.0501464419066906, 0.04751264303922653, 0.027365924790501595, 0.0561474971473217, 0.03319565951824188]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9041034579277039, 0.09589655697345734, 0.0, 0.0, 0.0, 0.0], [0.5862311720848083, 0.07199837267398834, 0.3417704999446869, 0.0, 0.0, 0.0], [0.38789618015289307, 0.046608004719018936, 0.20278996229171753, 0.36270585656166077, 0.0, 0.0], [0.2665242850780487, 0.02453301101922989, 0.12211935967206955, 0.20041202008724213, 0.3864113688468933, 0.0], [0.2335740178823471, 0.020537324249744415, 0.09610337764024734, 0.13062253594398499, 0.2299049347639084, 0.28925779461860657]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639912247657776, 0.03600878641009331, 0.0, 0.0, 0.0, 0.0], [0.7075549960136414, 0.25427794456481934, 0.03816710785031319, 0.0, 0.0, 0.0], [0.25665247440338135, 0.2058933526277542, 0.016656633466482162, 0.520797610282898, 0.0, 0.0], [0.10379401594400406, 0.04639105498790741, 0.00869862549006939, 0.7866847515106201, 0.05443162843585014, 0.0], [0.22143346071243286, 0.033797502517700195, 0.029023950919508934, 0.541292130947113, 0.15286119282245636, 0.0215916745364666]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891703724861145, 0.010829640552401543, 0.0, 0.0, 0.0, 0.0], [0.7913153767585754, 0.12309644371271133, 0.08558813482522964, 0.0, 0.0, 0.0], [0.2954603135585785, 0.15808317065238953, 0.4217239022254944, 0.12473267316818237, 0.0, 0.0], [0.23441052436828613, 0.09886535257101059, 0.33160147070884705, 0.19713923335075378, 0.13798339664936066, 0.0], [0.19728368520736694, 0.05741846561431885, 0.06909020990133286, 0.1646982580423355, 0.2797277271747589, 0.2317817062139511]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9359127879142761, 0.06408718228340149, 0.0, 0.0, 0.0, 0.0], [0.7888625860214233, 0.0867348238825798, 0.12440251559019089, 0.0, 0.0, 0.0], [0.6535119414329529, 0.07573550939559937, 0.09732570499181747, 0.17342683672904968, 0.0, 0.0], [0.5222766995429993, 0.05827884376049042, 0.09920478612184525, 0.1702084094285965, 0.15003129839897156, 0.0], [0.4108838438987732, 0.047306086868047714, 0.07265683263540268, 0.10560749471187592, 0.10550011694431305, 0.25804561376571655]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9683831930160522, 0.031616728752851486, 0.0, 0.0, 0.0, 0.0], [0.896539568901062, 0.03887060284614563, 0.06458979845046997, 0.0, 0.0, 0.0], [0.8264952898025513, 0.03213466331362724, 0.051967162638902664, 0.08940286934375763, 0.0, 0.0], [0.7718175053596497, 0.030402861535549164, 0.04582742601633072, 0.07118464261293411, 0.08076757192611694, 0.0], [0.7292331457138062, 0.021699873730540276, 0.033074744045734406, 0.04720093309879303, 0.06474556028842926, 0.10404572635889053]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979567527770996, 0.0020432821474969387, 0.0, 0.0, 0.0, 0.0], [0.9552940726280212, 0.008025333285331726, 0.03668065741658211, 0.0, 0.0, 0.0], [0.9254711270332336, 0.00275557953864336, 0.002062988467514515, 0.06971034407615662, 0.0, 0.0], [0.8660575151443481, 0.003888372564688325, 0.0006785987643525004, 0.0006981458282098174, 0.12867732346057892, 0.0], [0.8455936908721924, 0.0037804015446454287, 0.0002534226223360747, 6.0270347603363916e-05, 0.00011820702638942748, 0.150193989276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9262455701828003, 0.07375437766313553, 0.0, 0.0, 0.0, 0.0], [0.7717147469520569, 0.1624203771352768, 0.06586486846208572, 0.0, 0.0, 0.0], [0.8167640566825867, 0.07807155698537827, 0.06324019283056259, 0.04192415252327919, 0.0, 0.0], [0.6867176294326782, 0.07755190879106522, 0.1005692109465599, 0.05955103039741516, 0.07561030983924866, 0.0], [0.6421176791191101, 0.11014875024557114, 0.07688140124082565, 0.05403309315443039, 0.1033359244465828, 0.013483130373060703]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9395952820777893, 0.06040477007627487, 0.0, 0.0, 0.0, 0.0], [0.2300449162721634, 0.6617392301559448, 0.10821589082479477, 0.0, 0.0, 0.0], [0.26702260971069336, 0.3607953190803528, 0.3249623775482178, 0.04721971973776817, 0.0, 0.0], [0.5952032804489136, 0.12269210070371628, 0.06302005052566528, 0.08916770666837692, 0.12991684675216675, 0.0], [0.1028466448187828, 0.029380010440945625, 0.01373895350843668, 0.04586036503314972, 0.7698498964309692, 0.03832418471574783]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9040982723236084, 0.0959017425775528, 0.0, 0.0, 0.0, 0.0], [0.357235312461853, 0.6274638175964355, 0.015300935134291649, 0.0, 0.0, 0.0], [0.5918000936508179, 0.27640387415885925, 0.1047603040933609, 0.027035744860768318, 0.0, 0.0], [0.7254436612129211, 0.04983079805970192, 0.01498265191912651, 0.17781229317188263, 0.03193056955933571, 0.0], [0.7612767219543457, 0.06158901005983353, 0.005942164920270443, 0.01642669551074505, 0.12677954137325287, 0.02798592671751976]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9947587847709656, 0.005241230130195618, 0.0, 0.0, 0.0, 0.0], [0.9632415771484375, 0.017816435545682907, 0.018942005932331085, 0.0, 0.0, 0.0], [0.9671075940132141, 0.008509604260325432, 0.00856224074959755, 0.015820497646927834, 0.0, 0.0], [0.9340998530387878, 0.011952383443713188, 0.020180154591798782, 0.026750771328806877, 0.007016891613602638, 0.0], [0.9587240219116211, 0.0046570925042033195, 0.003326763864606619, 0.006545269396156073, 0.010182411409914494, 0.016564475372433662]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769991040229797, 0.02300085686147213, 0.0, 0.0, 0.0, 0.0], [0.791760265827179, 0.1753326654434204, 0.032907143235206604, 0.0, 0.0, 0.0], [0.7949190735816956, 0.10531850159168243, 0.04021850973367691, 0.05954388156533241, 0.0, 0.0], [0.7097727060317993, 0.10552525520324707, 0.06597540527582169, 0.05765572935342789, 0.06107088923454285, 0.0], [0.7506603598594666, 0.026514418423175812, 0.02157600037753582, 0.034296635538339615, 0.08494450896978378, 0.08200804889202118]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.983751654624939, 0.01624833419919014, 0.0, 0.0, 0.0, 0.0], [0.5615487694740295, 0.08956838399171829, 0.3488827645778656, 0.0, 0.0, 0.0], [0.3292914927005768, 0.024114832282066345, 0.5428048968315125, 0.10378878563642502, 0.0, 0.0], [0.34330493211746216, 0.013086404651403427, 0.5121950507164001, 0.11146263778209686, 0.01995093934237957, 0.0], [0.4792822599411011, 0.017333481460809708, 0.11805278062820435, 0.06130271032452583, 0.20071901381015778, 0.12330969423055649]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9908847212791443, 0.009115317836403847, 0.0, 0.0, 0.0, 0.0], [0.5282679200172424, 0.3292284309864044, 0.14250363409519196, 0.0, 0.0, 0.0], [0.48788732290267944, 0.233686164021492, 0.17577986419200897, 0.10264668613672256, 0.0, 0.0], [0.31444936990737915, 0.18065084517002106, 0.16871292889118195, 0.09506543725728989, 0.24112139642238617, 0.0], [0.5168782472610474, 0.03589695692062378, 0.026187807321548462, 0.040397077798843384, 0.18791691958904266, 0.19272294640541077]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8750306367874146, 0.12496936321258545, 0.0, 0.0, 0.0, 0.0], [0.4550616145133972, 0.4900423586368561, 0.05489595979452133, 0.0, 0.0, 0.0], [0.29337263107299805, 0.5449898838996887, 0.09444298595190048, 0.06719449907541275, 0.0, 0.0], [0.4897097647190094, 0.2720980942249298, 0.06861936300992966, 0.14694873988628387, 0.02262401394546032, 0.0], [0.4729084074497223, 0.08103056997060776, 0.01605204865336418, 0.3067218065261841, 0.10120705515146255, 0.022080253809690475]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9630220532417297, 0.03697795420885086, 0.0, 0.0, 0.0, 0.0], [0.7557179927825928, 0.1643647700548172, 0.07991719990968704, 0.0, 0.0, 0.0], [0.6947709918022156, 0.08409848809242249, 0.06382585316896439, 0.15730471909046173, 0.0, 0.0], [0.5821163654327393, 0.03297785297036171, 0.07936536520719528, 0.1944136917591095, 0.11112680286169052, 0.0], [0.5974539518356323, 0.04261082038283348, 0.06919696182012558, 0.14563480019569397, 0.12481768429279327, 0.02028587833046913]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9957822561264038, 0.004217814654111862, 0.0, 0.0, 0.0, 0.0], [0.9312829971313477, 0.010560262948274612, 0.058156777173280716, 0.0, 0.0, 0.0], [0.8435326218605042, 0.015694990754127502, 0.045751187950372696, 0.09502114355564117, 0.0, 0.0], [0.7724104523658752, 0.011981194838881493, 0.035046011209487915, 0.038767579942941666, 0.14179480075836182, 0.0], [0.7642906904220581, 0.00986877828836441, 0.008122753351926804, 0.013314371928572655, 0.04824399948120117, 0.15615928173065186]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9701175093650818, 0.029882490634918213, 0.0, 0.0, 0.0, 0.0], [0.6563995480537415, 0.22506217658519745, 0.11853817850351334, 0.0, 0.0, 0.0], [0.6958061456680298, 0.1470184177160263, 0.07145984470844269, 0.08571554720401764, 0.0, 0.0], [0.6353272795677185, 0.13460662961006165, 0.030994117259979248, 0.05691640451550484, 0.1421555131673813, 0.0], [0.6779407262802124, 0.05365397781133652, 0.018006201833486557, 0.06284506618976593, 0.11038198322057724, 0.07717210799455643]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9822332859039307, 0.01776670478284359, 0.0, 0.0, 0.0, 0.0], [0.9037660956382751, 0.06541543453931808, 0.030818426981568336, 0.0, 0.0, 0.0], [0.8119187355041504, 0.036790236830711365, 0.06056087091565132, 0.09073004871606827, 0.0, 0.0], [0.40546342730522156, 0.10383892804384232, 0.10211260616779327, 0.3543427288532257, 0.03424236178398132, 0.0], [0.22824503481388092, 0.017278604209423065, 0.05055435746908188, 0.6015732288360596, 0.09411737322807312, 0.008231470361351967]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873148202896118, 0.012685154564678669, 0.0, 0.0, 0.0, 0.0], [0.354457825422287, 0.531760036945343, 0.11378218978643417, 0.0, 0.0, 0.0], [0.07823401689529419, 0.7221351265907288, 0.10936620831489563, 0.09026475995779037, 0.0, 0.0], [0.21968045830726624, 0.40484222769737244, 0.1235806941986084, 0.2001885175704956, 0.051708124577999115, 0.0], [0.36089763045310974, 0.1045902818441391, 0.06983798742294312, 0.29764822125434875, 0.13869896531105042, 0.028326835483312607]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9732161164283752, 0.026783881708979607, 0.0, 0.0, 0.0, 0.0], [0.9167553782463074, 0.06145283579826355, 0.02179183065891266, 0.0, 0.0, 0.0], [0.8543079495429993, 0.08049621433019638, 0.03033490665256977, 0.03486097976565361, 0.0, 0.0], [0.8919220566749573, 0.04280766472220421, 0.022044911980628967, 0.023470569401979446, 0.01975482702255249, 0.0], [0.8116769194602966, 0.03413527458906174, 0.03567648306488991, 0.04748578742146492, 0.025397121906280518, 0.045628443360328674]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9502761363983154, 0.04972386732697487, 0.0, 0.0, 0.0, 0.0], [0.7637453675270081, 0.20073619484901428, 0.03551839664578438, 0.0, 0.0, 0.0], [0.6279090046882629, 0.037681419402360916, 0.19945412874221802, 0.13495545089244843, 0.0, 0.0], [0.6397066712379456, 0.027007192373275757, 0.09081999957561493, 0.20653821527957916, 0.03592786192893982, 0.0], [0.45594266057014465, 0.02164124697446823, 0.12939591705799103, 0.21800900995731354, 0.10379799455404282, 0.07121305912733078]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9498405456542969, 0.05015944689512253, 0.0, 0.0, 0.0, 0.0], [0.868872344493866, 0.08722198009490967, 0.04390567168593407, 0.0, 0.0, 0.0], [0.6937950849533081, 0.06359203159809113, 0.09179060161113739, 0.1508222371339798, 0.0, 0.0], [0.7266592383384705, 0.04389891028404236, 0.046839818358421326, 0.09851827472448349, 0.08408384770154953, 0.0], [0.7848992943763733, 0.037147846072912216, 0.012907843105494976, 0.010539384558796883, 0.12079204618930817, 0.03371358662843704]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891054034233093, 0.01089461799710989, 0.0, 0.0, 0.0, 0.0], [0.8929510116577148, 0.08700131624937057, 0.020047729834914207, 0.0, 0.0, 0.0], [0.7891115546226501, 0.09797312319278717, 0.0863322764635086, 0.026583032682538033, 0.0, 0.0], [0.8850637674331665, 0.0364501029253006, 0.053954314440488815, 0.012377242557704449, 0.01215458381921053, 0.0], [0.6861329078674316, 0.057203810662031174, 0.0116362813860178, 0.02166050672531128, 0.17488020658493042, 0.048486314713954926]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9396193027496338, 0.06038068234920502, 0.0, 0.0, 0.0, 0.0], [0.7851802110671997, 0.19751375913619995, 0.01730601117014885, 0.0, 0.0, 0.0], [0.7660512328147888, 0.15444660186767578, 0.0318828783929348, 0.04761931300163269, 0.0, 0.0], [0.7035237550735474, 0.05171409994363785, 0.0776095986366272, 0.15339010953903198, 0.013762442395091057, 0.0], [0.7121893763542175, 0.04994220659136772, 0.03772540017962456, 0.08649095147848129, 0.06541402637958527, 0.04823804274201393]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.974072277545929, 0.025927739217877388, 0.0, 0.0, 0.0, 0.0], [0.7925394773483276, 0.011715609580278397, 0.19574493169784546, 0.0, 0.0, 0.0], [0.5106772184371948, 0.007296781521290541, 0.03962003067135811, 0.44240593910217285, 0.0, 0.0], [0.5862484574317932, 0.01209966279566288, 0.024585144594311714, 0.06737807393074036, 0.30968865752220154, 0.0], [0.3019636273384094, 0.007724008988589048, 0.011518117040395737, 0.04694722592830658, 0.2214670479297638, 0.41037997603416443]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9744554162025452, 0.02554461918771267, 0.0, 0.0, 0.0, 0.0], [0.9769195318222046, 0.015048524364829063, 0.00803192425519228, 0.0, 0.0, 0.0], [0.9060619473457336, 0.025875424966216087, 0.02595478855073452, 0.04210774973034859, 0.0, 0.0], [0.9400084614753723, 0.00555663276463747, 0.005828281864523888, 0.031757764518260956, 0.016849050298333168, 0.0], [0.9105738401412964, 0.0019752182997763157, 0.008646713569760323, 0.013360859826207161, 0.03543969988822937, 0.030003685504198074]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9791666865348816, 0.02083336003124714, 0.0, 0.0, 0.0, 0.0], [0.8444863557815552, 0.13507835566997528, 0.020435357466340065, 0.0, 0.0, 0.0], [0.7903085947036743, 0.14559166133403778, 0.037530042231082916, 0.026569712907075882, 0.0, 0.0], [0.7298934459686279, 0.05649632588028908, 0.03273547440767288, 0.10400423407554626, 0.07687054574489594, 0.0], [0.5684199929237366, 0.04388810321688652, 0.026293331757187843, 0.08117101341485977, 0.2431478202342987, 0.037079744040966034]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9499868154525757, 0.050013184547424316, 0.0, 0.0, 0.0, 0.0], [0.9336171746253967, 0.05848868191242218, 0.007894255220890045, 0.0, 0.0, 0.0], [0.7897834181785583, 0.11071813106536865, 0.05360180512070656, 0.045896656811237335, 0.0, 0.0], [0.8859304785728455, 0.05752970278263092, 0.013743232935667038, 0.0033877352252602577, 0.03940894827246666, 0.0], [0.9337607622146606, 0.026470666751265526, 0.004523405339568853, 0.0061904969625175, 0.01413296815007925, 0.014921738766133785]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.0, 1.8521648659231005e-09, 0.0, 0.0, 0.0, 0.0], [6.6758143475453835e-06, 0.9999804496765137, 1.284119480260415e-05, 0.0, 0.0, 0.0], [2.219406880499264e-08, 2.668508969350114e-09, 0.9999971389770508, 2.813725814121426e-06, 0.0, 0.0], [1.0145217856916133e-06, 4.4640319885047575e-08, 0.0003535518771968782, 0.9993677735328674, 0.00027762516401708126, 0.0], [9.436730286083161e-10, 1.3820468207359493e-11, 5.017710691390675e-10, 2.9652009736480522e-09, 0.9999971389770508, 2.864445150407846e-06]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9948632121086121, 0.00513678602874279, 0.0, 0.0, 0.0, 0.0], [0.9274216890335083, 0.018323812633752823, 0.05425459146499634, 0.0, 0.0, 0.0], [0.9678993225097656, 0.0041434201411902905, 0.004314461722970009, 0.023642871528863907, 0.0, 0.0], [0.899907648563385, 0.0014671608805656433, 0.00029133600764907897, 0.002585014561191201, 0.09574881941080093, 0.0], [0.9386116862297058, 0.00022248241293709725, 0.000614665390457958, 0.0015495705883949995, 0.030689310282468796, 0.02831241302192211]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9999959468841553, 4.042735781695228e-06, 0.0, 0.0, 0.0, 0.0], [0.9982761144638062, 3.2613615985610522e-06, 0.001720665255561471, 0.0, 0.0, 0.0], [0.9998809099197388, 5.328835683826583e-08, 6.376233727678482e-07, 0.00011847116547869518, 0.0, 0.0], [0.9996154308319092, 3.4731792197817413e-07, 3.892067468314053e-08, 4.468433303372876e-07, 0.00038369561661966145, 0.0], [0.9994840621948242, 1.6550142589721872e-08, 2.871539450666205e-08, 1.0638284493325045e-06, 0.00021266516705509275, 0.0003021192387677729]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9514135718345642, 0.048586443066596985, 0.0, 0.0, 0.0, 0.0], [0.5749930143356323, 0.39028263092041016, 0.03472428396344185, 0.0, 0.0, 0.0], [0.744231641292572, 0.1752413809299469, 0.07564764469861984, 0.004879307933151722, 0.0, 0.0], [0.5232048034667969, 0.09429354965686798, 0.11381909251213074, 0.199794739484787, 0.06888782978057861, 0.0], [0.4747248888015747, 0.05636602267622948, 0.045303650200366974, 0.06967318058013916, 0.3098025918006897, 0.044129688292741776]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8734647631645203, 0.12653520703315735, 0.0, 0.0, 0.0, 0.0], [0.6097909212112427, 0.3541729748249054, 0.03603604808449745, 0.0, 0.0, 0.0], [0.4598419666290283, 0.3869785964488983, 0.09960105270147324, 0.05357832461595535, 0.0, 0.0], [0.5722200274467468, 0.23636257648468018, 0.08344569057226181, 0.06921947747468948, 0.038752224296331406, 0.0], [0.514358401298523, 0.16722990572452545, 0.09019368141889572, 0.0765446200966835, 0.10578057914972305, 0.045892804861068726]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981228768825531, 0.018771234899759293, 0.0, 0.0, 0.0, 0.0], [0.6142938137054443, 0.35039809346199036, 0.03530815243721008, 0.0, 0.0, 0.0], [0.5770687460899353, 0.32858437299728394, 0.05508258566260338, 0.03926431015133858, 0.0, 0.0], [0.17188090085983276, 0.011042523197829723, 0.05457884445786476, 0.7326592803001404, 0.02983849309384823, 0.0], [0.3783024251461029, 0.017070062458515167, 0.021754097193479538, 0.4409683644771576, 0.06093815341591835, 0.08096696436405182]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9923112392425537, 0.007688735146075487, 0.0, 0.0, 0.0, 0.0], [0.9498787522315979, 0.016709759831428528, 0.033411506563425064, 0.0, 0.0, 0.0], [0.9961295127868652, 0.0008787286351434886, 0.0006868178606964648, 0.002304842695593834, 0.0, 0.0], [0.9935757517814636, 0.003263507504016161, 0.0009993839776143432, 0.0002793238090816885, 0.001882061013020575, 0.0], [0.9907532930374146, 0.00021344359265640378, 0.0004595248610712588, 0.0007905631209723651, 0.004424731712788343, 0.003358363639563322]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9647740125656128, 0.03522596135735512, 0.0, 0.0, 0.0, 0.0], [0.8194130659103394, 0.1365436464548111, 0.044043250381946564, 0.0, 0.0, 0.0], [0.7584242820739746, 0.006878956686705351, 0.20653414726257324, 0.028162650763988495, 0.0, 0.0], [0.5298121571540833, 0.002678812248632312, 0.07857993245124817, 0.3598378598690033, 0.029091132804751396, 0.0], [0.754442572593689, 0.0003678239299915731, 0.001971345627680421, 0.003240057732909918, 0.19423308968544006, 0.04574507102370262]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9749132394790649, 0.025086797773838043, 0.0, 0.0, 0.0, 0.0], [0.9306469559669495, 0.057056792080402374, 0.012296278029680252, 0.0, 0.0, 0.0], [0.9305251836776733, 0.05277099460363388, 0.011119411326944828, 0.005584418307989836, 0.0, 0.0], [0.8863323926925659, 0.012924155220389366, 0.017724651843309402, 0.06150183826684952, 0.021517043933272362, 0.0], [0.7916852831840515, 0.015036062337458134, 0.031747858971357346, 0.033922016620635986, 0.03707969933748245, 0.09052912145853043]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.960849940776825, 0.039150021970272064, 0.0, 0.0, 0.0, 0.0], [0.9121273159980774, 0.022576453164219856, 0.06529619544744492, 0.0, 0.0, 0.0], [0.9364106059074402, 0.01558446604758501, 0.024544961750507355, 0.023459969088435173, 0.0, 0.0], [0.9454619288444519, 0.006762297358363867, 0.022026164457201958, 0.009137828834354877, 0.0166118573397398, 0.0], [0.8346167802810669, 0.0018816972151398659, 0.005609032232314348, 0.01887362077832222, 0.1244916245341301, 0.014527121558785439]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964227080345154, 0.003577286144718528, 0.0, 0.0, 0.0, 0.0], [0.9713928699493408, 0.024453043937683105, 0.004154064226895571, 0.0, 0.0, 0.0], [0.9735792279243469, 0.01900341734290123, 0.0036644143983721733, 0.003752920078113675, 0.0, 0.0], [0.9586310982704163, 0.007116200402379036, 0.009218422695994377, 0.02272571064531803, 0.0023084881249815226, 0.0], [0.973607063293457, 0.008490582928061485, 0.0032512503676116467, 0.0036064486484974623, 0.004877470899373293, 0.006167230196297169]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97598797082901, 0.024012047797441483, 0.0, 0.0, 0.0, 0.0], [0.946063756942749, 0.04211379587650299, 0.011822430416941643, 0.0, 0.0, 0.0], [0.8446813225746155, 0.04293113946914673, 0.05218193307518959, 0.0602056086063385, 0.0, 0.0], [0.9378372430801392, 0.03354857116937637, 0.008826462551951408, 0.0028792363591492176, 0.016908491030335426, 0.0], [0.8124933838844299, 0.026967575773596764, 0.05999194458127022, 0.03445727750658989, 0.011011868715286255, 0.05507809296250343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9001206755638123, 0.09987929463386536, 0.0, 0.0, 0.0, 0.0], [0.6271925568580627, 0.07988723367452621, 0.2929201126098633, 0.0, 0.0, 0.0], [0.7624078989028931, 0.02734428085386753, 0.03867945075035095, 0.17156831920146942, 0.0, 0.0], [0.7995962500572205, 0.014336277730762959, 0.014375667087733746, 0.025438543409109116, 0.1462532877922058, 0.0], [0.7851975560188293, 0.04204043000936508, 0.025253599509596825, 0.02908393368124962, 0.029306255280971527, 0.08911828696727753]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954467415809631, 0.004553207661956549, 0.0, 0.0, 0.0, 0.0], [0.9356004595756531, 0.04476727917790413, 0.019632209092378616, 0.0, 0.0, 0.0], [0.5605551600456238, 0.09862001240253448, 0.2998324930667877, 0.04099231958389282, 0.0, 0.0], [0.589370608329773, 0.1100098192691803, 0.08033646643161774, 0.16754046082496643, 0.0527426153421402, 0.0], [0.223059743642807, 0.056807976216077805, 0.05467941239476204, 0.24733929336071014, 0.3111240565776825, 0.10698945075273514]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9301448464393616, 0.06985516101121902, 0.0, 0.0, 0.0, 0.0], [0.8936478495597839, 0.08535714447498322, 0.020995058119297028, 0.0, 0.0, 0.0], [0.8404536843299866, 0.10619213432073593, 0.023636789992451668, 0.029717300087213516, 0.0, 0.0], [0.8927381634712219, 0.02478480525314808, 0.008319051936268806, 0.05165467783808708, 0.022503303363919258, 0.0], [0.8646609783172607, 0.009503169916570187, 0.00243298034183681, 0.04796737805008888, 0.042732127010822296, 0.032703299075365067]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859625697135925, 0.014037402346730232, 0.0, 0.0, 0.0, 0.0], [0.9702038764953613, 0.01680702343583107, 0.012989096343517303, 0.0, 0.0, 0.0], [0.9524770379066467, 0.016064444556832314, 0.013456214219331741, 0.018002301454544067, 0.0, 0.0], [0.9332928657531738, 0.018971974030137062, 0.020146867260336876, 0.017023736611008644, 0.010564574971795082, 0.0], [0.9113593697547913, 0.012528610415756702, 0.022096145898103714, 0.017518552020192146, 0.01851789653301239, 0.01797933503985405]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9681769013404846, 0.03182312101125717, 0.0, 0.0, 0.0, 0.0], [0.9096419215202332, 0.07916687428951263, 0.011191246099770069, 0.0, 0.0, 0.0], [0.8379933834075928, 0.13078251481056213, 0.012140998616814613, 0.019083019345998764, 0.0, 0.0], [0.9116523265838623, 0.05451972037553787, 0.00949938129633665, 0.007465865928679705, 0.016862763091921806, 0.0], [0.8510287404060364, 0.07338211685419083, 0.008022502064704895, 0.009083151817321777, 0.04261011257767677, 0.015873325988650322]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9799023866653442, 0.02009769156575203, 0.0, 0.0, 0.0, 0.0], [0.9558742642402649, 0.029063263908028603, 0.015062462538480759, 0.0, 0.0, 0.0], [0.7943130731582642, 0.06074107438325882, 0.0690767914056778, 0.0758691281080246, 0.0, 0.0], [0.549432098865509, 0.031547121703624725, 0.054820187389850616, 0.0578807033598423, 0.306319922208786, 0.0], [0.645397961139679, 0.010770927183330059, 0.017528077587485313, 0.02157982438802719, 0.24958311021327972, 0.05514014512300491]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9506808519363403, 0.049319129437208176, 0.0, 0.0, 0.0, 0.0], [0.8553208708763123, 0.09256317466497421, 0.05211597681045532, 0.0, 0.0, 0.0], [0.8508525490760803, 0.04734596610069275, 0.04417724534869194, 0.057624202221632004, 0.0, 0.0], [0.7697128653526306, 0.027885887771844864, 0.031017253175377846, 0.06842502951622009, 0.10295900702476501, 0.0], [0.7931906580924988, 0.040521834045648575, 0.029241926968097687, 0.04478121176362038, 0.04894692823290825, 0.04331746697425842]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9770310521125793, 0.022968946024775505, 0.0, 0.0, 0.0, 0.0], [0.9429818391799927, 0.017321474850177765, 0.03969673812389374, 0.0, 0.0, 0.0], [0.9144347906112671, 0.008583571761846542, 0.013035777024924755, 0.06394589692354202, 0.0, 0.0], [0.9222431182861328, 0.003644028678536415, 0.003740261774510145, 0.010410326533019543, 0.05996226519346237, 0.0], [0.9198879599571228, 0.0030822642147541046, 0.0034827410709112883, 0.0042068008333444595, 0.021254301071166992, 0.04808594286441803]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9774582386016846, 0.02254181168973446, 0.0, 0.0, 0.0, 0.0], [0.8929324150085449, 0.07475479692220688, 0.032312821596860886, 0.0, 0.0, 0.0], [0.8423513174057007, 0.05980277061462402, 0.037400756031274796, 0.060445044189691544, 0.0, 0.0], [0.7674633264541626, 0.035363439470529556, 0.04215509817004204, 0.06658629328012466, 0.08843192458152771, 0.0], [0.6182613372802734, 0.016110623255372047, 0.020167628303170204, 0.03868889808654785, 0.2314697951078415, 0.0753016546368599]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9634856581687927, 0.036514345556497574, 0.0, 0.0, 0.0, 0.0], [0.43639448285102844, 0.5226364731788635, 0.040969014167785645, 0.0, 0.0, 0.0], [0.36086124181747437, 0.35129690170288086, 0.2655107080936432, 0.02233118563890457, 0.0, 0.0], [0.39429229497909546, 0.021704625338315964, 0.07794322818517685, 0.3716890215873718, 0.13437077403068542, 0.0], [0.6310721635818481, 0.016983957961201668, 0.025942014530301094, 0.08615921437740326, 0.21831963956356049, 0.021522965282201767]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9988250136375427, 0.0011750265257433057, 0.0, 0.0, 0.0, 0.0], [0.9944871068000793, 0.0004826387739740312, 0.005030233412981033, 0.0, 0.0, 0.0], [0.9981209635734558, 2.7051630240748636e-05, 0.00011307397653581575, 0.0017389433924108744, 0.0, 0.0], [0.9982239603996277, 6.836584361735731e-05, 0.00010199935059063137, 6.028370262356475e-05, 0.001545313629321754, 0.0], [0.9982888102531433, 1.0552178082434693e-06, 3.278081203461625e-05, 0.0001303895260207355, 0.0006605856469832361, 0.0008863675757311285]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9936710596084595, 0.006328933872282505, 0.0, 0.0, 0.0, 0.0], [0.9727688431739807, 0.0018561349716037512, 0.025375083088874817, 0.0, 0.0, 0.0], [0.9724301099777222, 0.00195861142128706, 0.011192452162504196, 0.014418844133615494, 0.0, 0.0], [0.9782041311264038, 0.0009589171386323869, 0.001870649284683168, 0.006326562725007534, 0.01263973768800497, 0.0], [0.9592596888542175, 0.002455513458698988, 0.0016124167013913393, 0.005019667558372021, 0.006687135435640812, 0.024965722113847733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9629000425338745, 0.03709990158677101, 0.0, 0.0, 0.0, 0.0], [0.36802080273628235, 0.6152244210243225, 0.016754813492298126, 0.0, 0.0, 0.0], [0.31735214591026306, 0.6140003800392151, 0.053751468658447266, 0.014896074309945107, 0.0, 0.0], [0.4898741543292999, 0.21071387827396393, 0.04693012312054634, 0.20700415968894958, 0.04547771066427231, 0.0], [0.48774242401123047, 0.1769523322582245, 0.06915220618247986, 0.09849317371845245, 0.12091425806283951, 0.04674568027257919]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9794419407844543, 0.02055807039141655, 0.0, 0.0, 0.0, 0.0], [0.6677890419960022, 0.31032490730285645, 0.021886074915528297, 0.0, 0.0, 0.0], [0.711875855922699, 0.11108537018299103, 0.14187364280223846, 0.03516502305865288, 0.0, 0.0], [0.4501466155052185, 0.04036048427224159, 0.04045815393328667, 0.3885694742202759, 0.0804651752114296, 0.0], [0.49346205592155457, 0.013696958310902119, 0.00812679436057806, 0.13074517250061035, 0.30861401557922363, 0.04535504803061485]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9846054315567017, 0.015394601970911026, 0.0, 0.0, 0.0, 0.0], [0.9806739091873169, 0.007713806349784136, 0.011612341739237309, 0.0, 0.0, 0.0], [0.9326630234718323, 0.01957845874130726, 0.024103542789816856, 0.02365495264530182, 0.0, 0.0], [0.9422018527984619, 0.0009538960293866694, 0.0010898001492023468, 0.0031933635473251343, 0.05256103351712227, 0.0], [0.9352929592132568, 0.0010279365815222263, 0.004444439895451069, 0.0016371352830901742, 0.010590966790914536, 0.047006525099277496]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985783100128174, 0.0014216770650818944, 0.0, 0.0, 0.0, 0.0], [0.9893348813056946, 0.0011178908171132207, 0.009547143243253231, 0.0, 0.0, 0.0], [0.9979978203773499, 7.997050124686211e-05, 0.00013218825915828347, 0.0017900299280881882, 0.0, 0.0], [0.9986976385116577, 4.1044117097044364e-05, 3.868332441925304e-06, 2.3676169803366065e-05, 0.0012337109073996544, 0.0], [0.9971562623977661, 1.8522179743740708e-05, 1.8826593759513344e-06, 2.7900150598725304e-05, 0.0006533485138788819, 0.002141993958503008]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9768233299255371, 0.023176709190011024, 0.0, 0.0, 0.0, 0.0], [0.919467568397522, 0.050882015377283096, 0.029650306329131126, 0.0, 0.0, 0.0], [0.8474555015563965, 0.06100171059370041, 0.04372366517782211, 0.047819141298532486, 0.0, 0.0], [0.8011622428894043, 0.041867081075906754, 0.04375801607966423, 0.041894808411598206, 0.07131782919168472, 0.0], [0.8031870126724243, 0.024504918605089188, 0.01732351817190647, 0.0474439337849617, 0.06109936535358429, 0.04644131660461426]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9829428195953369, 0.01705716922879219, 0.0, 0.0, 0.0, 0.0], [0.8863736391067505, 0.09492656588554382, 0.018699748441576958, 0.0, 0.0, 0.0], [0.9231085777282715, 0.03696347773075104, 0.03219832852482796, 0.007729670964181423, 0.0, 0.0], [0.9068530201911926, 0.01604657620191574, 0.014310453087091446, 0.04543779417872429, 0.01735224947333336, 0.0], [0.6555968523025513, 0.05091030150651932, 0.028384873643517494, 0.125655397772789, 0.1054687574505806, 0.03398393467068672]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9502318501472473, 0.04976809769868851, 0.0, 0.0, 0.0, 0.0], [0.8829867243766785, 0.1000962033867836, 0.01691715233027935, 0.0, 0.0, 0.0], [0.8057456016540527, 0.14463560283184052, 0.03018922172486782, 0.01942945271730423, 0.0, 0.0], [0.8706232905387878, 0.03244058042764664, 0.02695159800350666, 0.04410297051072121, 0.025881517678499222, 0.0], [0.6883644461631775, 0.00968148186802864, 0.016449354588985443, 0.09871124476194382, 0.08971203118562698, 0.09708139300346375]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9792683124542236, 0.020731747150421143, 0.0, 0.0, 0.0, 0.0], [0.9523284435272217, 0.025933869183063507, 0.02173773944377899, 0.0, 0.0, 0.0], [0.9144353866577148, 0.017671285197138786, 0.02235853485763073, 0.045534808188676834, 0.0, 0.0], [0.9448293447494507, 0.006467591971158981, 0.006386054679751396, 0.03263082355260849, 0.009686139412224293, 0.0], [0.9347907304763794, 0.007862498052418232, 0.007788168732076883, 0.02143275924026966, 0.008491129614412785, 0.0196347888559103]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9833701848983765, 0.01662975549697876, 0.0, 0.0, 0.0, 0.0], [0.9631112813949585, 0.009229970164597034, 0.02765871398150921, 0.0, 0.0, 0.0], [0.9706628322601318, 0.004149409011006355, 0.006813114508986473, 0.01837460696697235, 0.0, 0.0], [0.987951934337616, 0.0021658889017999172, 0.0003490102826617658, 0.001583811710588634, 0.007949435152113438, 0.0], [0.945794939994812, 0.014583585783839226, 0.00036529666977003217, 0.0009569558897055686, 0.01362166740000248, 0.024677546694874763]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9878059029579163, 0.012194057926535606, 0.0, 0.0, 0.0, 0.0], [0.8710366487503052, 0.0944816991686821, 0.03448163717985153, 0.0, 0.0, 0.0], [0.6309780478477478, 0.11090393364429474, 0.1923023909330368, 0.06581564992666245, 0.0, 0.0], [0.5360495448112488, 0.04618944227695465, 0.13605298101902008, 0.2645546793937683, 0.01715332642197609, 0.0], [0.8287513852119446, 0.02373281493782997, 0.02008042484521866, 0.07245299965143204, 0.0304313562810421, 0.02455105073750019]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8995685577392578, 0.10043143481016159, 0.0, 0.0, 0.0, 0.0], [0.270342618227005, 0.6504339575767517, 0.07922345399856567, 0.0, 0.0, 0.0], [0.20541740953922272, 0.5892512798309326, 0.180857852101326, 0.02447350136935711, 0.0, 0.0], [0.5573872327804565, 0.17741312086582184, 0.0880676656961441, 0.09881818294525146, 0.07831382751464844, 0.0], [0.5922901034355164, 0.08700644969940186, 0.056432779878377914, 0.05685899406671524, 0.1218155100941658, 0.08559606969356537]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9316376447677612, 0.068362295627594, 0.0, 0.0, 0.0, 0.0], [0.9572947025299072, 0.02624349854886532, 0.016461782157421112, 0.0, 0.0, 0.0], [0.9880544543266296, 0.004273331258445978, 0.0029545812867581844, 0.004717642907053232, 0.0, 0.0], [0.99403977394104, 0.0009413447696715593, 0.00047398178139701486, 0.00011646964412648231, 0.004428428132086992, 0.0], [0.9806035161018372, 2.5468958483543247e-05, 0.00016239396063610911, 0.0001476412871852517, 0.001344241201877594, 0.017716852948069572]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.993178129196167, 0.006821870803833008, 0.0, 0.0, 0.0, 0.0], [0.9756524562835693, 0.013184132054448128, 0.011163455434143543, 0.0, 0.0, 0.0], [0.9418967962265015, 0.004721763078123331, 0.0023818148765712976, 0.0509997121989727, 0.0, 0.0], [0.9905040860176086, 0.0022848148364573717, 6.19848578935489e-05, 0.0005984479794278741, 0.006550685502588749, 0.0], [0.9697661399841309, 0.0008878838270902634, 0.0002346673863939941, 0.0017040770035237074, 0.004128350410610437, 0.02327890321612358]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9716230630874634, 0.0283768679946661, 0.0, 0.0, 0.0, 0.0], [0.9223619699478149, 0.028907284140586853, 0.048730771988630295, 0.0, 0.0, 0.0], [0.8426315784454346, 0.02387218549847603, 0.04748142883181572, 0.0860147550702095, 0.0, 0.0], [0.8521116375923157, 0.020744290202856064, 0.044946324080228806, 0.05765015631914139, 0.024547478184103966, 0.0], [0.8800724744796753, 0.02244853600859642, 0.01823573186993599, 0.01925482228398323, 0.015854286029934883, 0.04413414001464844]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9412722587585449, 0.05872778967022896, 0.0, 0.0, 0.0, 0.0], [0.9163140654563904, 0.05759185180068016, 0.026094017550349236, 0.0, 0.0, 0.0], [0.8392423987388611, 0.05769050121307373, 0.013829062692821026, 0.08923796564340591, 0.0, 0.0], [0.8987160921096802, 0.013477860949933529, 0.0003456451231613755, 0.0032987555023282766, 0.08416163921356201, 0.0], [0.8701690435409546, 0.0027008659671992064, 0.0014349978882819414, 0.0056661684066057205, 0.08874314278364182, 0.031285740435123444]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9656725525856018, 0.03432748094201088, 0.0, 0.0, 0.0, 0.0], [0.9178615212440491, 0.062257926911115646, 0.019880497828125954, 0.0, 0.0, 0.0], [0.8233146667480469, 0.06282396614551544, 0.03670436143875122, 0.07715708762407303, 0.0, 0.0], [0.8501746654510498, 0.03816923871636391, 0.031964968889951706, 0.05160145089030266, 0.028089623898267746, 0.0], [0.6572403907775879, 0.0587739571928978, 0.04336011782288551, 0.09013210982084274, 0.08146586269140244, 0.06902758777141571]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9162064790725708, 0.083793506026268, 0.0, 0.0, 0.0, 0.0], [0.9451773762702942, 0.040992818772792816, 0.013829846866428852, 0.0, 0.0, 0.0], [0.8928354978561401, 0.05368676036596298, 0.01759696751832962, 0.035880785435438156, 0.0, 0.0], [0.8337051868438721, 0.04799617826938629, 0.033513251692056656, 0.0468086414039135, 0.03797678276896477, 0.0], [0.8167194724082947, 0.06337127834558487, 0.013286269269883633, 0.020469754934310913, 0.025292271748185158, 0.06086096167564392]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9525133371353149, 0.04748661443591118, 0.0, 0.0, 0.0, 0.0], [0.30198633670806885, 0.6520944833755493, 0.045919183641672134, 0.0, 0.0, 0.0], [0.2855835556983948, 0.5569521188735962, 0.14447391033172607, 0.012990497052669525, 0.0, 0.0], [0.8438040018081665, 0.03225131705403328, 0.03954288363456726, 0.06848171353340149, 0.01592012494802475, 0.0], [0.6664936542510986, 0.06095915287733078, 0.04064338654279709, 0.06804481148719788, 0.09186346083879471, 0.07199545204639435]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9682656526565552, 0.031734418123960495, 0.0, 0.0, 0.0, 0.0], [0.7385215759277344, 0.22856871783733368, 0.03290969505906105, 0.0, 0.0, 0.0], [0.5946669578552246, 0.23033174872398376, 0.1486765593290329, 0.02632465586066246, 0.0, 0.0], [0.6339261531829834, 0.058130085468292236, 0.09654293209314346, 0.14291933178901672, 0.06848142296075821, 0.0], [0.40375757217407227, 0.08945360779762268, 0.07635073363780975, 0.255870521068573, 0.14330360293388367, 0.031263869255781174]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869793653488159, 0.013020600192248821, 0.0, 0.0, 0.0, 0.0], [0.863138735294342, 0.11056642234325409, 0.026294825598597527, 0.0, 0.0, 0.0], [0.9488077759742737, 0.02861507050693035, 0.006535581778734922, 0.016041584312915802, 0.0, 0.0], [0.9672170877456665, 0.006604989990592003, 0.0004517161869443953, 0.004844421520829201, 0.020881719887256622, 0.0], [0.9354623556137085, 0.0204780176281929, 0.001170022296719253, 0.007056924514472485, 0.016318105161190033, 0.01951460912823677]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9846673011779785, 0.01533269602805376, 0.0, 0.0, 0.0, 0.0], [0.9052745699882507, 0.08373622596263885, 0.010989241302013397, 0.0, 0.0, 0.0], [0.814594030380249, 0.042837489396333694, 0.10568274557590485, 0.0368858203291893, 0.0, 0.0], [0.23519699275493622, 0.012018428184092045, 0.05280094966292381, 0.6516199111938477, 0.048363760113716125, 0.0], [0.31818681955337524, 0.01863236539065838, 0.03948163613677025, 0.3755547106266022, 0.20787166059017181, 0.040272727608680725]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9811733365058899, 0.018826739862561226, 0.0, 0.0, 0.0, 0.0], [0.8618937730789185, 0.0647917315363884, 0.07331447303295135, 0.0, 0.0, 0.0], [0.7664536833763123, 0.07330439984798431, 0.10353528708219528, 0.05670657008886337, 0.0, 0.0], [0.8128500580787659, 0.03215484321117401, 0.05900546908378601, 0.05416510999202728, 0.04182443022727966, 0.0], [0.8687862157821655, 0.02698776312172413, 0.02046988718211651, 0.01629730872809887, 0.032183703035116196, 0.035275109112262726]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9264581799507141, 0.07354183495044708, 0.0, 0.0, 0.0, 0.0], [0.8403534293174744, 0.06373776495456696, 0.0959087386727333, 0.0, 0.0, 0.0], [0.7330995202064514, 0.06451123207807541, 0.10380081832408905, 0.09858846664428711, 0.0, 0.0], [0.9143611788749695, 0.008257807232439518, 0.007320408709347248, 0.01796630211174488, 0.05209442973136902, 0.0], [0.8971914052963257, 0.008555497974157333, 0.007019439246505499, 0.014860527589917183, 0.03399759903550148, 0.03837534412741661]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9180346727371216, 0.08196533471345901, 0.0, 0.0, 0.0, 0.0], [0.8328660726547241, 0.12199065834283829, 0.045143257826566696, 0.0, 0.0, 0.0], [0.7994153499603271, 0.08744156360626221, 0.03605782240629196, 0.07708528637886047, 0.0, 0.0], [0.8809850215911865, 0.02074967324733734, 0.020554570481181145, 0.01712082512676716, 0.060589876025915146, 0.0], [0.7453039288520813, 0.04433402791619301, 0.022549191489815712, 0.033152736723423004, 0.03357040509581566, 0.12108971923589706]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9867060780525208, 0.013293899595737457, 0.0, 0.0, 0.0, 0.0], [0.982177734375, 0.012414142489433289, 0.0054081035777926445, 0.0, 0.0, 0.0], [0.96304851770401, 0.015290786512196064, 0.010345697402954102, 0.011314956471323967, 0.0, 0.0], [0.9213568568229675, 0.014132496900856495, 0.017639189958572388, 0.016567690297961235, 0.030303703621029854, 0.0], [0.9373330473899841, 0.009064311161637306, 0.007548339664936066, 0.006576436571776867, 0.011827593669295311, 0.027650361880660057]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9951004385948181, 0.00489962799474597, 0.0, 0.0, 0.0, 0.0], [0.9476007223129272, 0.041407953947782516, 0.010991264134645462, 0.0, 0.0, 0.0], [0.9142175316810608, 0.023523805662989616, 0.03914501518011093, 0.02311362884938717, 0.0, 0.0], [0.9534734487533569, 0.008932976052165031, 0.015272831544280052, 0.007908296771347523, 0.01441233605146408, 0.0], [0.9427104592323303, 0.008233043365180492, 0.0046509732492268085, 0.004178094677627087, 0.0054635051637887955, 0.03476400673389435]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9543375372886658, 0.04566241055727005, 0.0, 0.0, 0.0, 0.0], [0.9696040749549866, 0.019547609612345695, 0.010848305188119411, 0.0, 0.0, 0.0], [0.9710449576377869, 0.012425409629940987, 0.008068887516856194, 0.008460711687803268, 0.0, 0.0], [0.9726192951202393, 0.002697661053389311, 0.0004483141237869859, 0.0013814771082252264, 0.022853175178170204, 0.0], [0.9675466418266296, 0.009613418020308018, 0.0032030276488512754, 0.004248825367540121, 0.0074422359466552734, 0.007945875637233257]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9887008666992188, 0.011299148201942444, 0.0, 0.0, 0.0, 0.0], [0.9382631182670593, 0.04204246401786804, 0.019694389775395393, 0.0, 0.0, 0.0], [0.8351995944976807, 0.034878626465797424, 0.051344748586416245, 0.07857709378004074, 0.0, 0.0], [0.9042678475379944, 0.010541587136685848, 0.016426727175712585, 0.02592189982533455, 0.04284190386533737, 0.0], [0.8913140296936035, 0.008912663906812668, 0.005010687280446291, 0.00817559752613306, 0.013514691032469273, 0.07307220995426178]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8693915009498596, 0.13060854375362396, 0.0, 0.0, 0.0, 0.0], [0.3507999777793884, 0.6063508987426758, 0.0428491048514843, 0.0, 0.0, 0.0], [0.35475727915763855, 0.3502015769481659, 0.24722374975681305, 0.04781733825802803, 0.0, 0.0], [0.35370829701423645, 0.03527732938528061, 0.09567102789878845, 0.4497946500778198, 0.06554868817329407, 0.0], [0.41325902938842773, 0.09055528044700623, 0.05286572128534317, 0.17467984557151794, 0.17384859919548035, 0.0947914645075798]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9629756212234497, 0.03702434524893761, 0.0, 0.0, 0.0, 0.0], [0.9756426811218262, 0.019658586010336876, 0.00469872634857893, 0.0, 0.0, 0.0], [0.9775736331939697, 0.013286303728818893, 0.002559040440246463, 0.006581075489521027, 0.0, 0.0], [0.9870141744613647, 0.007388271391391754, 0.000957920856308192, 0.001831832341849804, 0.0028077582828700542, 0.0], [0.9409245848655701, 0.016633689403533936, 0.0022979099303483963, 0.005890651606023312, 0.005512924864888191, 0.02874019183218479]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628270864486694, 0.03717285394668579, 0.0, 0.0, 0.0, 0.0], [0.9582237601280212, 0.024641864001750946, 0.017134418711066246, 0.0, 0.0, 0.0], [0.9351300001144409, 0.015331589616835117, 0.014810995198786259, 0.03472748026251793, 0.0, 0.0], [0.9225173592567444, 0.010528748854994774, 0.011010152287781239, 0.019440028816461563, 0.03650377318263054, 0.0], [0.8420167565345764, 0.04357195645570755, 0.007488266099244356, 0.01496153511106968, 0.023852793499827385, 0.06810864061117172]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9926387071609497, 0.007361333351582289, 0.0, 0.0, 0.0, 0.0], [0.9957393407821655, 0.003346970770508051, 0.00091368897119537, 0.0, 0.0, 0.0], [0.9869900345802307, 0.0019747859332710505, 0.0015245546819642186, 0.009510686621069908, 0.0, 0.0], [0.9933527708053589, 0.001020328258164227, 0.00034337403485551476, 0.001029114704579115, 0.004254393745213747, 0.0], [0.9749016761779785, 0.0004348019137978554, 0.000430655520176515, 0.0012364384019747376, 0.001534773618914187, 0.021461637690663338]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9897475242614746, 0.010252486914396286, 0.0, 0.0, 0.0, 0.0], [0.9790639281272888, 0.016509056091308594, 0.00442709494382143, 0.0, 0.0, 0.0], [0.9521436095237732, 0.02943236567080021, 0.008943174965679646, 0.009480932727456093, 0.0, 0.0], [0.9395942091941833, 0.021510910242795944, 0.010278534144163132, 0.004555221181362867, 0.024061065167188644, 0.0], [0.9205074310302734, 0.01615365780889988, 0.010818609036505222, 0.01664443127810955, 0.014566411264240742, 0.02130948379635811]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9898502826690674, 0.01014977227896452, 0.0, 0.0, 0.0, 0.0], [0.9820910096168518, 0.0069075338542461395, 0.011001535691320896, 0.0, 0.0, 0.0], [0.9684997200965881, 0.0089876102283597, 0.015342569909989834, 0.007170100696384907, 0.0, 0.0], [0.9274120330810547, 0.009485269896686077, 0.02206611819565296, 0.03222890570759773, 0.008807665668427944, 0.0], [0.9006660580635071, 0.02162371575832367, 0.013808215968310833, 0.009843839332461357, 0.008521351031959057, 0.04553685709834099]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954444169998169, 0.004555594641715288, 0.0, 0.0, 0.0, 0.0], [0.995254397392273, 0.002460234332829714, 0.0022854835260659456, 0.0, 0.0, 0.0], [0.9862446188926697, 0.0015168144600465894, 0.0040722922421991825, 0.008166342042386532, 0.0, 0.0], [0.9889963865280151, 0.0012260458897799253, 0.0007996382773853838, 0.0006774249486625195, 0.008300590328872204, 0.0], [0.9865202903747559, 0.0003942706680390984, 0.0009571767877787352, 0.0004954360192641616, 0.0009604952065274119, 0.010672261007130146]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9821295142173767, 0.017870519310235977, 0.0, 0.0, 0.0, 0.0], [0.7489445805549622, 0.22002629935741425, 0.03102915547788143, 0.0, 0.0, 0.0], [0.2854783236980438, 0.21125619113445282, 0.47871607542037964, 0.024549338966608047, 0.0, 0.0], [0.8056642413139343, 0.02697453647851944, 0.04302811250090599, 0.06993735581636429, 0.05439591035246849, 0.0], [0.330721914768219, 0.022326510399580002, 0.016627032309770584, 0.080194391310215, 0.41574740409851074, 0.13438260555267334]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9697746634483337, 0.030225370079278946, 0.0, 0.0, 0.0, 0.0], [0.9800565838813782, 0.01501889992505312, 0.004924521781504154, 0.0, 0.0, 0.0], [0.9237861037254333, 0.052764732390642166, 0.006302398629486561, 0.017146700993180275, 0.0, 0.0], [0.9451842904090881, 0.03618054464459419, 0.001989212818443775, 0.0039587244391441345, 0.01268730964511633, 0.0], [0.9633325934410095, 0.018663031980395317, 0.0030418476089835167, 0.007070912979543209, 0.0050094276666641235, 0.0028820668812841177]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.987324595451355, 0.012675448320806026, 0.0, 0.0, 0.0, 0.0], [0.9904569983482361, 0.005541946738958359, 0.004001116845756769, 0.0, 0.0, 0.0], [0.9814971685409546, 0.004653455223888159, 0.00372527539730072, 0.010124039836227894, 0.0, 0.0], [0.9744364619255066, 0.004632262047380209, 0.002379997167736292, 0.006518092937767506, 0.012033062055706978, 0.0], [0.9624499678611755, 0.0033743453677743673, 0.0013198527740314603, 0.0017274925485253334, 0.002944671083241701, 0.028183719143271446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9807673692703247, 0.019232625141739845, 0.0, 0.0, 0.0, 0.0], [0.9664247035980225, 0.0154139194637537, 0.01816137507557869, 0.0, 0.0, 0.0], [0.9632683396339417, 0.004538131412118673, 0.0029253889806568623, 0.029268164187669754, 0.0, 0.0], [0.9562349915504456, 0.0012223643716424704, 0.0005304075893945992, 0.008671474643051624, 0.033340781927108765, 0.0], [0.9657101035118103, 0.0009808274917304516, 0.0016686212038621306, 0.0026348272804170847, 0.005866338964551687, 0.02313929796218872]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639714956283569, 0.036028504371643066, 0.0, 0.0, 0.0, 0.0], [0.9562800526618958, 0.033733122050762177, 0.009986838325858116, 0.0, 0.0, 0.0], [0.853999674320221, 0.08073031902313232, 0.03334450349211693, 0.031925544142723083, 0.0, 0.0], [0.9547489881515503, 0.0096050426363945, 0.00414617033675313, 0.002013325458392501, 0.02948645129799843, 0.0], [0.933113694190979, 0.028699707239866257, 0.005477478262037039, 0.006368071772158146, 0.012613070197403431, 0.013728084973990917]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9392993450164795, 0.060700662434101105, 0.0, 0.0, 0.0, 0.0], [0.9298389554023743, 0.061895497143268585, 0.008265494368970394, 0.0, 0.0, 0.0], [0.8471820950508118, 0.09035064280033112, 0.017636114731431007, 0.04483110085129738, 0.0, 0.0], [0.8857702016830444, 0.03918180242180824, 0.007867713458836079, 0.022765880450606346, 0.0444144681096077, 0.0], [0.8563281893730164, 0.10088981688022614, 0.00653143459931016, 0.008485895581543446, 0.0073684388771653175, 0.020396238192915916]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8353262543678284, 0.16467373073101044, 0.0, 0.0, 0.0, 0.0], [0.6160860061645508, 0.3137648403644562, 0.07014918327331543, 0.0, 0.0, 0.0], [0.34316354990005493, 0.27584975957870483, 0.11966027319431305, 0.261326402425766, 0.0, 0.0], [0.5908167958259583, 0.05029088631272316, 0.04166605323553085, 0.2199493646621704, 0.09727690368890762, 0.0], [0.8481416702270508, 0.0631808340549469, 0.014733636751770973, 0.05526723712682724, 0.009014973416924477, 0.009661651216447353]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9627320766448975, 0.03726791962981224, 0.0, 0.0, 0.0, 0.0], [0.7757521867752075, 0.17996282875537872, 0.044284969568252563, 0.0, 0.0, 0.0], [0.6317057609558105, 0.24380743503570557, 0.10925643891096115, 0.015230290591716766, 0.0, 0.0], [0.9539909958839417, 0.01818232797086239, 0.011601817794144154, 0.01229910645633936, 0.003925777971744537, 0.0], [0.40357086062431335, 0.14237552881240845, 0.056611865758895874, 0.19757317006587982, 0.09299169480800629, 0.10687682777643204]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9802619218826294, 0.01973811723291874, 0.0, 0.0, 0.0, 0.0], [0.9873908162117004, 0.007800452411174774, 0.004808676429092884, 0.0, 0.0, 0.0], [0.9283919334411621, 0.00830123946070671, 0.01330562774091959, 0.05000118911266327, 0.0, 0.0], [0.8981053829193115, 0.015591327100992203, 0.01017757598310709, 0.03998705744743347, 0.03613865002989769, 0.0], [0.975350022315979, 0.0003543295315466821, 0.0005866009742021561, 0.0011877480428665876, 0.0010750859510153532, 0.021446382626891136]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9295330047607422, 0.0704670250415802, 0.0, 0.0, 0.0, 0.0], [0.9361505508422852, 0.04116692394018173, 0.022682547569274902, 0.0, 0.0, 0.0], [0.8486821055412292, 0.05802806839346886, 0.024856165051460266, 0.0684337243437767, 0.0, 0.0], [0.8661178946495056, 0.02232472598552704, 0.010369138792157173, 0.026001976802945137, 0.07518625259399414, 0.0], [0.8074422478675842, 0.04438253492116928, 0.01849709264934063, 0.03357789292931557, 0.0185612291097641, 0.07753899693489075]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9680536389350891, 0.031946372240781784, 0.0, 0.0, 0.0, 0.0], [0.9693689942359924, 0.025684954598546028, 0.004946068394929171, 0.0, 0.0, 0.0], [0.9620568156242371, 0.02255246415734291, 0.005471329670399427, 0.00991946179419756, 0.0, 0.0], [0.9727528095245361, 0.010137127712368965, 0.0007573263137601316, 0.0028828911017626524, 0.013469814322888851, 0.0], [0.9624637365341187, 0.00311090424656868, 0.00100075569935143, 0.0019475899171084166, 0.008266208693385124, 0.023210842162370682]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8542505502700806, 0.14574943482875824, 0.0, 0.0, 0.0, 0.0], [0.9725967645645142, 0.014116342179477215, 0.013286862522363663, 0.0, 0.0, 0.0], [0.9257622957229614, 0.03257261589169502, 0.014612065628170967, 0.02705308608710766, 0.0, 0.0], [0.7923429012298584, 0.027305010706186295, 0.01880667358636856, 0.1385413110256195, 0.02300402708351612, 0.0], [0.6152051091194153, 0.026655299589037895, 0.029352962970733643, 0.0559089221060276, 0.11611286550760269, 0.15676498413085938]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9804654121398926, 0.019534558057785034, 0.0, 0.0, 0.0, 0.0], [0.9882452487945557, 0.0075094737112522125, 0.004245333839207888, 0.0, 0.0, 0.0], [0.9584206342697144, 0.010963579639792442, 0.010456085205078125, 0.020159730687737465, 0.0, 0.0], [0.9604810476303101, 0.007182636763900518, 0.003072344930842519, 0.0068989223800599575, 0.022365106269717216, 0.0], [0.966888964176178, 0.0032812876161187887, 0.005500528495758772, 0.004234079271554947, 0.005038034170866013, 0.015057181939482689]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9498193860054016, 0.050180625170469284, 0.0, 0.0, 0.0, 0.0], [0.9781363606452942, 0.016430046409368515, 0.005433559417724609, 0.0, 0.0, 0.0], [0.8618695139884949, 0.03609364852309227, 0.07555560022592545, 0.026481211185455322, 0.0, 0.0], [0.5449836850166321, 0.015411168336868286, 0.02351653389632702, 0.25743618607521057, 0.1586524099111557, 0.0], [0.9571872353553772, 0.0030803889967501163, 0.001444686553440988, 0.006861576810479164, 0.014818795025348663, 0.01660728268325329]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6156561374664307, 0.3843438923358917, 0.0, 0.0, 0.0, 0.0], [0.36760637164115906, 0.4281637370586395, 0.20422989130020142, 0.0, 0.0, 0.0], [0.16471558809280396, 0.4136791527271271, 0.25092360377311707, 0.1706816852092743, 0.0, 0.0], [0.4184460937976837, 0.15247608721256256, 0.10305383056402206, 0.11071506887674332, 0.21530893445014954, 0.0], [0.19686944782733917, 0.2014620304107666, 0.12827250361442566, 0.0920325294137001, 0.09167566150426865, 0.28968778252601624]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9027365446090698, 0.09726346284151077, 0.0, 0.0, 0.0, 0.0], [0.9736634492874146, 0.014004308730363846, 0.012332303449511528, 0.0, 0.0, 0.0], [0.8504457473754883, 0.05690574273467064, 0.03206087276339531, 0.06058763712644577, 0.0, 0.0], [0.766120970249176, 0.035303935408592224, 0.03433044254779816, 0.09675208479166031, 0.06749249249696732, 0.0], [0.8650376796722412, 0.020085245370864868, 0.0114980423822999, 0.018558325245976448, 0.018430287018418312, 0.06639042496681213]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9653082489967346, 0.03469175100326538, 0.0, 0.0, 0.0, 0.0], [0.9816323518753052, 0.014176092110574245, 0.004191520158201456, 0.0, 0.0, 0.0], [0.9275255799293518, 0.0473722405731678, 0.011528268456459045, 0.013573979958891869, 0.0, 0.0], [0.9293115139007568, 0.025833338499069214, 0.007227125111967325, 0.01430062297731638, 0.023327331990003586, 0.0], [0.8895061612129211, 0.046896226704120636, 0.004717112518846989, 0.006286595948040485, 0.006090151146054268, 0.046503737568855286]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8938026428222656, 0.10619737207889557, 0.0, 0.0, 0.0, 0.0], [0.8221706748008728, 0.063044972717762, 0.11478440463542938, 0.0, 0.0, 0.0], [0.5047378540039062, 0.15375757217407227, 0.22770363092422485, 0.11380091309547424, 0.0, 0.0], [0.40820738673210144, 0.09066358208656311, 0.11696857213973999, 0.24553203582763672, 0.13862840831279755, 0.0], [0.7291041016578674, 0.06638885289430618, 0.023112762719392776, 0.03110307641327381, 0.05714317411184311, 0.09314802289009094]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9247532486915588, 0.07524676620960236, 0.0, 0.0, 0.0, 0.0], [0.8957375288009644, 0.06989562511444092, 0.03436676785349846, 0.0, 0.0, 0.0], [0.7924938201904297, 0.09601156413555145, 0.055091120302677155, 0.056403499096632004, 0.0, 0.0], [0.7891507148742676, 0.07880303263664246, 0.03840154409408569, 0.05396977812051773, 0.039674971252679825, 0.0], [0.780785858631134, 0.07993540912866592, 0.04253171756863594, 0.03234210982918739, 0.017816951498389244, 0.04658801481127739]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9480886459350586, 0.05191135033965111, 0.0, 0.0, 0.0, 0.0], [0.8636949062347412, 0.047562092542648315, 0.08874304592609406, 0.0, 0.0, 0.0], [0.9341371059417725, 0.022224130108952522, 0.022624509409070015, 0.021014362573623657, 0.0, 0.0], [0.9588144421577454, 0.008020910434424877, 0.004490078892558813, 0.00586229981854558, 0.022812405601143837, 0.0], [0.938591718673706, 0.02122773416340351, 0.0048724752850830555, 0.010940182954072952, 0.009524590335786343, 0.01484343409538269]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9763734340667725, 0.023626603186130524, 0.0, 0.0, 0.0, 0.0], [0.9884802103042603, 0.005189403425902128, 0.006330370437353849, 0.0, 0.0, 0.0], [0.9477092027664185, 0.017985165119171143, 0.0101566007360816, 0.024149015545845032, 0.0, 0.0], [0.967192530632019, 0.006552820093929768, 0.003322787582874298, 0.005563332699239254, 0.017368430271744728, 0.0], [0.9584562182426453, 0.007502953987568617, 0.0051363264210522175, 0.008071641437709332, 0.005997130647301674, 0.014835829846560955]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.884070873260498, 0.11592914164066315, 0.0, 0.0, 0.0, 0.0], [0.993125319480896, 0.0050708153285086155, 0.0018038052367046475, 0.0, 0.0, 0.0], [0.9534158110618591, 0.023829087615013123, 0.007748983334749937, 0.015006095170974731, 0.0, 0.0], [0.9151288270950317, 0.010873124934732914, 0.013190999627113342, 0.011050461791455746, 0.049756716936826706, 0.0], [0.8769674301147461, 0.03385210409760475, 0.008486478589475155, 0.009969156235456467, 0.03468579426407814, 0.036039117723703384]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003709519514814019, 0.999629020690918, 0.0, 0.0, 0.0, 0.0], [6.52503949822858e-05, 0.37378302216529846, 0.6261518001556396, 0.0, 0.0, 0.0], [4.606039874488488e-05, 0.21050886809825897, 0.411596417427063, 0.377848744392395, 0.0, 0.0], [4.7531026211800054e-05, 0.11616962403059006, 0.23264268040657043, 0.3985332250595093, 0.2526068687438965, 0.0], [1.2476449455789407e-06, 0.1481970250606537, 0.15813151001930237, 0.30074411630630493, 0.11939028650522232, 0.273535817861557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.971555769443512, 0.02844420075416565, 0.0, 0.0, 0.0, 0.0], [0.9529063701629639, 0.03233078494668007, 0.014762787148356438, 0.0, 0.0, 0.0], [0.9343128204345703, 0.023512953892350197, 0.020498033612966537, 0.021676233038306236, 0.0, 0.0], [0.9529678225517273, 0.00855142343789339, 0.004359327722340822, 0.008064564317464828, 0.026056958362460136, 0.0], [0.9653593897819519, 0.008487634360790253, 0.003499275306239724, 0.002721574855968356, 0.0032828773837536573, 0.016649343073368073]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8630780577659607, 0.13692186772823334, 0.0, 0.0, 0.0, 0.0], [0.7696157097816467, 0.08513346314430237, 0.14525090157985687, 0.0, 0.0, 0.0], [0.7133336067199707, 0.1017090305685997, 0.11931272596120834, 0.06564465165138245, 0.0, 0.0], [0.7186216711997986, 0.05444302782416344, 0.013868178240954876, 0.0780804380774498, 0.13498668372631073, 0.0], [0.7990152835845947, 0.058055855333805084, 0.009447006508708, 0.017770448699593544, 0.021138517186045647, 0.09457287937402725]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9518988728523254, 0.04810114949941635, 0.0, 0.0, 0.0, 0.0], [0.8580654263496399, 0.029445774853229523, 0.11248882114887238, 0.0, 0.0, 0.0], [0.6577739119529724, 0.08513441681861877, 0.12613077461719513, 0.1309608817100525, 0.0, 0.0], [0.8087368607521057, 0.03230159357190132, 0.018418151885271072, 0.06856140494346619, 0.07198190689086914, 0.0], [0.668330192565918, 0.13281351327896118, 0.021880602464079857, 0.02787742204964161, 0.049234092235565186, 0.09986425936222076]]]], \"left_text\": [\"No\", \",\", \" I\", \" am\", \" your\", \" father\"], \"right_text\": [\"No\", \",\", \" I\", \" am\", \" your\", \" father\"]}], \"default_filter\": \"0\", \"display_mode\": \"dark\", \"root_div_id\": \"bertviz-2fa5f9f15ab7437992290934271cfac7\", \"include_layers\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], \"include_heads\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], \"total_heads\": 12}; // HACK: {\"attention\": [{\"name\": null, \"attn\": [[[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.961219847202301, 0.038780149072408676, 0.0, 0.0, 0.0, 0.0], [0.7466980218887329, 0.11987308412790298, 0.1334288865327835, 0.0, 0.0, 0.0], [0.5885030031204224, 0.13792069256305695, 0.21213732659816742, 0.06143900007009506, 0.0, 0.0], [0.657085657119751, 0.08996297419071198, 0.1275128275156021, 0.08361561596393585, 0.04182284325361252, 0.0], [0.272887259721756, 0.11203360557556152, 0.16639851033687592, 0.08467116951942444, 0.16952727735042572, 0.19448219239711761]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010616553947329521, 0.9893833994865417, 0.0, 0.0, 0.0, 0.0], [0.002467755926772952, 0.008448011241853237, 0.9890841841697693, 0.0, 0.0, 0.0], [0.00012328448065090925, 0.0018733128672465682, 0.013126945123076439, 0.9848763942718506, 0.0, 0.0], [0.0010669530602172017, 0.0011366248363628983, 0.0030349865555763245, 0.00157350511290133, 0.9931879043579102, 0.0], [0.00019792001694440842, 0.0010528122074902058, 0.0015437351539731026, 0.0009642756194807589, 3.492440009722486e-05, 0.9962062835693359]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4757843017578125, 0.5242156982421875, 0.0, 0.0, 0.0, 0.0], [0.5906046032905579, 0.2486610859632492, 0.1607343554496765, 0.0, 0.0, 0.0], [0.5529289841651917, 0.18856702744960785, 0.14457568526268005, 0.11392833292484283, 0.0, 0.0], [0.4509407877922058, 0.16486789286136627, 0.17318037152290344, 0.11748013645410538, 0.0935308039188385, 0.0], [0.42572465538978577, 0.17328648269176483, 0.15651948750019073, 0.07022645324468613, 0.0808701142668724, 0.09337276965379715]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6133622527122498, 0.38663774728775024, 0.0, 0.0, 0.0, 0.0], [0.06098503991961479, 0.03253461420536041, 0.9064803123474121, 0.0, 0.0, 0.0], [0.0067170835100114346, 0.0004012887948192656, 0.7572956085205078, 0.2355860024690628, 0.0, 0.0], [0.03722767531871796, 0.002948855748400092, 0.10081084072589874, 0.04142269492149353, 0.8175899982452393, 0.0], [0.04989771544933319, 0.00030758281354792416, 0.0024198247119784355, 0.0034334948286414146, 0.0006823905860073864, 0.9432590007781982]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9489555954933167, 0.05104443058371544, 0.0, 0.0, 0.0, 0.0], [0.6821408867835999, 0.13952411711215973, 0.17833493649959564, 0.0, 0.0, 0.0], [0.20366327464580536, 0.056414876133203506, 0.06399297714233398, 0.6759288907051086, 0.0, 0.0], [0.34195470809936523, 0.067254438996315, 0.0792618840932846, 0.17836201190948486, 0.3331669569015503, 0.0], [0.09464021027088165, 0.007428212556988001, 0.006983972620218992, 0.007184368558228016, 0.018724262714385986, 0.8650389313697815]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33834612369537354, 0.6616538763046265, 0.0, 0.0, 0.0, 0.0], [0.07855986058712006, 0.006165443453937769, 0.9152746796607971, 0.0, 0.0, 0.0], [0.016775960102677345, 0.00040376983815804124, 0.0033404543064534664, 0.9794798493385315, 0.0, 0.0], [0.02760043926537037, 0.0004441527707967907, 0.0006541680195368826, 0.0002266185765620321, 0.971074640750885, 0.0], [0.010248193517327309, 3.70155721611809e-05, 0.00016064071678556502, 2.7341899112798274e-05, 1.0187313819187693e-05, 0.98951655626297]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9825032949447632, 0.01749667525291443, 0.0, 0.0, 0.0, 0.0], [0.8874197006225586, 0.05467936769127846, 0.05790088325738907, 0.0, 0.0, 0.0], [0.6849910020828247, 0.12280681729316711, 0.04972028359770775, 0.14248184859752655, 0.0, 0.0], [0.6015853881835938, 0.09881892055273056, 0.07070116698741913, 0.1665254533290863, 0.062369052320718765, 0.0], [0.32325053215026855, 0.12567415833473206, 0.04432179033756256, 0.07076980173587799, 0.06606654077768326, 0.3699170649051666]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9191646575927734, 0.08083534240722656, 0.0, 0.0, 0.0, 0.0], [0.4598640501499176, 0.39703115820884705, 0.14310480654239655, 0.0, 0.0, 0.0], [0.3003872334957123, 0.22181734442710876, 0.3816152513027191, 0.09618015587329865, 0.0, 0.0], [0.18963924050331116, 0.13763725757598877, 0.20173496007919312, 0.2363216131925583, 0.2346668690443039, 0.0], [0.15410453081130981, 0.09489504247903824, 0.11902561038732529, 0.1027795746922493, 0.4317219853401184, 0.09747327864170074]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.364999920129776, 0.6350001096725464, 0.0, 0.0, 0.0, 0.0], [0.24595212936401367, 0.5519201755523682, 0.20212772488594055, 0.0, 0.0, 0.0], [0.27213579416275024, 0.40738630294799805, 0.251862108707428, 0.06861570477485657, 0.0, 0.0], [0.1024254709482193, 0.16683608293533325, 0.5248051881790161, 0.05445462465286255, 0.15147872269153595, 0.0], [0.2502949833869934, 0.22198131680488586, 0.1889997124671936, 0.10677109658718109, 0.1303267925977707, 0.10162606090307236]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6990506649017334, 0.300949364900589, 0.0, 0.0, 0.0, 0.0], [0.5107942819595337, 0.294864296913147, 0.1943414956331253, 0.0, 0.0, 0.0], [0.46047067642211914, 0.28051918745040894, 0.19174796342849731, 0.06726215779781342, 0.0, 0.0], [0.3764842748641968, 0.21120664477348328, 0.20214536786079407, 0.10207021236419678, 0.10809348523616791, 0.0], [0.30138450860977173, 0.20456179976463318, 0.1825033277273178, 0.11019382625818253, 0.16291265189647675, 0.03844384104013443]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.713158369064331, 0.28684163093566895, 0.0, 0.0, 0.0, 0.0], [0.4058799147605896, 0.18063293397426605, 0.41348710656166077, 0.0, 0.0, 0.0], [0.26554614305496216, 0.16985861957073212, 0.3358593285083771, 0.22873590886592865, 0.0, 0.0], [0.31385400891304016, 0.1831670105457306, 0.14928355813026428, 0.05377669632434845, 0.2999187409877777, 0.0], [0.20466554164886475, 0.18731112778186798, 0.15959149599075317, 0.06381770968437195, 0.03642303869128227, 0.3481910526752472]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6586242318153381, 0.34137576818466187, 0.0, 0.0, 0.0, 0.0], [0.5917777419090271, 0.3160034716129303, 0.09221877157688141, 0.0, 0.0, 0.0], [0.5477151870727539, 0.23586955666542053, 0.06145601347088814, 0.15495926141738892, 0.0, 0.0], [0.4587061107158661, 0.22439998388290405, 0.07887422293424606, 0.0992034301161766, 0.13881628215312958, 0.0], [0.32743728160858154, 0.19600822031497955, 0.06805708259344101, 0.0892510712146759, 0.11618075519800186, 0.20306557416915894]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9961552023887634, 0.0038448390550911427, 0.0, 0.0, 0.0, 0.0], [0.8594854474067688, 0.06906113028526306, 0.07145342975854874, 0.0, 0.0, 0.0], [0.3800053596496582, 0.04127567261457443, 0.5496612191200256, 0.029057808220386505, 0.0, 0.0], [0.21445205807685852, 0.050887417048215866, 0.43174412846565247, 0.25869303941726685, 0.04422337934374809, 0.0], [0.11175268143415451, 0.01759307272732258, 0.027507441118359566, 0.04086774215102196, 0.7754669785499573, 0.02681216411292553]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9285967946052551, 0.07140323519706726, 0.0, 0.0, 0.0, 0.0], [0.6077285408973694, 0.31214284896850586, 0.08012856543064117, 0.0, 0.0, 0.0], [0.49429091811180115, 0.2850370407104492, 0.11849313229322433, 0.1021789088845253, 0.0, 0.0], [0.41838788986206055, 0.2311791181564331, 0.08340615779161453, 0.11365945637226105, 0.1533673107624054, 0.0], [0.4221586287021637, 0.1291714906692505, 0.08740919828414917, 0.10163754224777222, 0.21230259537696838, 0.04732052609324455]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9786475896835327, 0.02135237492620945, 0.0, 0.0, 0.0, 0.0], [0.7749121189117432, 0.0651036873459816, 0.15998415648937225, 0.0, 0.0, 0.0], [0.6484923958778381, 0.07483129948377609, 0.14751604199409485, 0.12916021049022675, 0.0, 0.0], [0.5224636793136597, 0.06921812891960144, 0.13823404908180237, 0.11106594651937485, 0.15901818871498108, 0.0], [0.39645177125930786, 0.07325821369886398, 0.1293814778327942, 0.10642431676387787, 0.14864003658294678, 0.14584419131278992]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5525906085968018, 0.44740942120552063, 0.0, 0.0, 0.0, 0.0], [0.5585009455680847, 0.2176259309053421, 0.22387315332889557, 0.0, 0.0, 0.0], [0.5143128633499146, 0.15964668989181519, 0.1549196094274521, 0.1711207777261734, 0.0, 0.0], [0.5039963126182556, 0.11401881277561188, 0.11974020302295685, 0.12552586197853088, 0.13671880960464478, 0.0], [0.5061841607093811, 0.08567394316196442, 0.0890301838517189, 0.09759820997714996, 0.1027572825551033, 0.11875618249177933]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9242545366287231, 0.07574543356895447, 0.0, 0.0, 0.0, 0.0], [0.8257424235343933, 0.07932537794113159, 0.0949321910738945, 0.0, 0.0, 0.0], [0.7306379675865173, 0.08571833372116089, 0.08043932169675827, 0.1032044067978859, 0.0, 0.0], [0.6383240818977356, 0.07886383682489395, 0.07815023511648178, 0.08758097141981125, 0.117080919444561, 0.0], [0.5552157163619995, 0.07409115880727768, 0.06834889203310013, 0.07778602838516235, 0.09999320656061172, 0.12456496059894562]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8578913807868958, 0.14210855960845947, 0.0, 0.0, 0.0, 0.0], [0.6423036456108093, 0.16629023849964142, 0.19140605628490448, 0.0, 0.0, 0.0], [0.5530978441238403, 0.10609277337789536, 0.07821263372898102, 0.2625967860221863, 0.0, 0.0], [0.4012168049812317, 0.1222359910607338, 0.19347327947616577, 0.14164631068706512, 0.14142754673957825, 0.0], [0.40212583541870117, 0.1845075935125351, 0.07516805827617645, 0.058490391820669174, 0.14446339011192322, 0.13524475693702698]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9791558980941772, 0.020844046026468277, 0.0, 0.0, 0.0, 0.0], [0.8829841613769531, 0.062332455068826675, 0.05468333512544632, 0.0, 0.0, 0.0], [0.8105455040931702, 0.08617087453603745, 0.07321774214506149, 0.030065858736634254, 0.0, 0.0], [0.6819811463356018, 0.04990815743803978, 0.08296556025743484, 0.08369529247283936, 0.10144980996847153, 0.0], [0.4056687355041504, 0.07337656617164612, 0.08601398020982742, 0.06170930340886116, 0.13226443529129028, 0.24096696078777313]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9670192003250122, 0.03298087790608406, 0.0, 0.0, 0.0, 0.0], [0.8449065089225769, 0.08514498919248581, 0.06994851678609848, 0.0, 0.0, 0.0], [0.7123571038246155, 0.07896049320697784, 0.055410757660865784, 0.1532716304063797, 0.0, 0.0], [0.640261173248291, 0.0739755630493164, 0.044393062591552734, 0.14322136342525482, 0.09814884513616562, 0.0], [0.5073902606964111, 0.07523062080144882, 0.07754648476839066, 0.11362482607364655, 0.13947948813438416, 0.08672824501991272]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8487567901611328, 0.15124322474002838, 0.0, 0.0, 0.0, 0.0], [0.8415648341178894, 0.1210724338889122, 0.03736277297139168, 0.0, 0.0, 0.0], [0.7505517601966858, 0.11348951607942581, 0.061799556016922, 0.07415911555290222, 0.0, 0.0], [0.6614715456962585, 0.10242648422718048, 0.052934277802705765, 0.0752972960472107, 0.10787033289670944, 0.0], [0.6014202237129211, 0.11340402066707611, 0.05631927400827408, 0.07096727937459946, 0.10906262695789337, 0.04882659390568733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9445484280586243, 0.05545159429311752, 0.0, 0.0, 0.0, 0.0], [0.8874567747116089, 0.05474221706390381, 0.057801052927970886, 0.0, 0.0, 0.0], [0.8281887769699097, 0.06895007193088531, 0.05903473123908043, 0.04382641240954399, 0.0, 0.0], [0.6429893970489502, 0.06747540831565857, 0.11629702895879745, 0.05417948588728905, 0.11905863881111145, 0.0], [0.7367823123931885, 0.056119006127119064, 0.06857286393642426, 0.034219563007354736, 0.07875377684831619, 0.025552386417984962]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0002913392090704292, 0.9997085928916931, 0.0, 0.0, 0.0, 0.0], [0.0007981202797964215, 0.5288337469100952, 0.4703681766986847, 0.0, 0.0, 0.0], [0.0007648473256267607, 0.3451983630657196, 0.3085266649723053, 0.3455101251602173, 0.0, 0.0], [0.0010283150477334857, 0.24135911464691162, 0.23320142924785614, 0.2555714249610901, 0.26883965730667114, 0.0], [0.0009746808791533113, 0.17789694666862488, 0.1674315482378006, 0.18587610125541687, 0.18734434247016907, 0.2804763615131378]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8244927525520325, 0.17550718784332275, 0.0, 0.0, 0.0, 0.0], [0.12386851012706757, 0.044499825686216354, 0.8316316604614258, 0.0, 0.0, 0.0], [0.07924355566501617, 0.012965913861989975, 0.0015277150087058544, 0.9062628149986267, 0.0, 0.0], [0.08806417882442474, 0.02134108357131481, 0.0028886189684271812, 0.002845388138666749, 0.8848607540130615, 0.0], [0.09983226656913757, 0.03363394737243652, 0.005499998107552528, 0.00243305298499763, 0.0015082467580214143, 0.8570924997329712]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9646892547607422, 0.03531072288751602, 0.0, 0.0, 0.0, 0.0], [0.7529155015945435, 0.08733480423688889, 0.15974965691566467, 0.0, 0.0, 0.0], [0.42022842168807983, 0.09195113927125931, 0.23549841344356537, 0.25232207775115967, 0.0, 0.0], [0.3084895610809326, 0.05908143147826195, 0.3839128613471985, 0.15659146010875702, 0.09192463010549545, 0.0], [0.447904109954834, 0.04329312965273857, 0.07969177514314651, 0.1108192726969719, 0.2212459146976471, 0.09704578667879105]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.991096019744873, 0.008904017508029938, 0.0, 0.0, 0.0, 0.0], [0.9697675704956055, 0.026084477081894875, 0.004147926811128855, 0.0, 0.0, 0.0], [0.9082902073860168, 0.03320597484707832, 0.009421153925359249, 0.04908263683319092, 0.0, 0.0], [0.8949136137962341, 0.055445361882448196, 0.005577605217695236, 0.03150679171085358, 0.012556544505059719, 0.0], [0.8497737050056458, 0.028890114277601242, 0.0036648025270551443, 0.03751996532082558, 0.038428016006946564, 0.041723549365997314]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9984525442123413, 0.0015474484534934163, 0.0, 0.0, 0.0, 0.0], [0.4894773066043854, 0.48122045397758484, 0.02930225059390068, 0.0, 0.0, 0.0], [0.11772146075963974, 0.13121247291564941, 0.67023104429245, 0.08083496242761612, 0.0, 0.0], [0.13043726980686188, 0.0406869612634182, 0.2652043104171753, 0.41143471002578735, 0.15223680436611176, 0.0], [0.12661851942539215, 0.03275134041905403, 0.03567883372306824, 0.06039177253842354, 0.6021823287010193, 0.14237718284130096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9805176854133606, 0.019482314586639404, 0.0, 0.0, 0.0, 0.0], [0.7948848009109497, 0.12061930447816849, 0.0844959020614624, 0.0, 0.0, 0.0], [0.5612354874610901, 0.15743164718151093, 0.2033972144126892, 0.07793562114238739, 0.0, 0.0], [0.4258382320404053, 0.10742025077342987, 0.15123654901981354, 0.08754990994930267, 0.22795499861240387, 0.0], [0.24752630293369293, 0.024188317358493805, 0.030395260080695152, 0.08586960285902023, 0.5714336633682251, 0.04058684781193733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9887767434120178, 0.011223198845982552, 0.0, 0.0, 0.0, 0.0], [0.7572691440582275, 0.22317387163639069, 0.0195570457726717, 0.0, 0.0, 0.0], [0.5341880321502686, 0.22107607126235962, 0.17621813714504242, 0.0685177892446518, 0.0, 0.0], [0.17095306515693665, 0.08229431509971619, 0.5760217905044556, 0.11097602546215057, 0.0597548633813858, 0.0], [0.2487107813358307, 0.08880821615457535, 0.08980182558298111, 0.09729332476854324, 0.4413091838359833, 0.03407657518982887]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.84221351146698, 0.1577865481376648, 0.0, 0.0, 0.0, 0.0], [0.46841251850128174, 0.4610540270805359, 0.07053346186876297, 0.0, 0.0, 0.0], [0.25881412625312805, 0.46358928084373474, 0.18503478169441223, 0.09256181120872498, 0.0, 0.0], [0.18399572372436523, 0.29154250025749207, 0.17031069099903107, 0.2717295289039612, 0.08242153376340866, 0.0], [0.16469858586788177, 0.24727003276348114, 0.08770552277565002, 0.2257496416568756, 0.1774536818265915, 0.09712250530719757]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9919946193695068, 0.008005395531654358, 0.0, 0.0, 0.0, 0.0], [0.9068723320960999, 0.044065166264772415, 0.049062490463256836, 0.0, 0.0, 0.0], [0.8582221865653992, 0.055348340421915054, 0.040419381111860275, 0.046010058373212814, 0.0, 0.0], [0.7855251431465149, 0.041242457926273346, 0.08369290828704834, 0.04887617751955986, 0.04066329449415207, 0.0], [0.7856318354606628, 0.0501464419066906, 0.04751264303922653, 0.027365924790501595, 0.0561474971473217, 0.03319565951824188]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9041034579277039, 0.09589655697345734, 0.0, 0.0, 0.0, 0.0], [0.5862311720848083, 0.07199837267398834, 0.3417704999446869, 0.0, 0.0, 0.0], [0.38789618015289307, 0.046608004719018936, 0.20278996229171753, 0.36270585656166077, 0.0, 0.0], [0.2665242850780487, 0.02453301101922989, 0.12211935967206955, 0.20041202008724213, 0.3864113688468933, 0.0], [0.2335740178823471, 0.020537324249744415, 0.09610337764024734, 0.13062253594398499, 0.2299049347639084, 0.28925779461860657]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639912247657776, 0.03600878641009331, 0.0, 0.0, 0.0, 0.0], [0.7075549960136414, 0.25427794456481934, 0.03816710785031319, 0.0, 0.0, 0.0], [0.25665247440338135, 0.2058933526277542, 0.016656633466482162, 0.520797610282898, 0.0, 0.0], [0.10379401594400406, 0.04639105498790741, 0.00869862549006939, 0.7866847515106201, 0.05443162843585014, 0.0], [0.22143346071243286, 0.033797502517700195, 0.029023950919508934, 0.541292130947113, 0.15286119282245636, 0.0215916745364666]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891703724861145, 0.010829640552401543, 0.0, 0.0, 0.0, 0.0], [0.7913153767585754, 0.12309644371271133, 0.08558813482522964, 0.0, 0.0, 0.0], [0.2954603135585785, 0.15808317065238953, 0.4217239022254944, 0.12473267316818237, 0.0, 0.0], [0.23441052436828613, 0.09886535257101059, 0.33160147070884705, 0.19713923335075378, 0.13798339664936066, 0.0], [0.19728368520736694, 0.05741846561431885, 0.06909020990133286, 0.1646982580423355, 0.2797277271747589, 0.2317817062139511]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9359127879142761, 0.06408718228340149, 0.0, 0.0, 0.0, 0.0], [0.7888625860214233, 0.0867348238825798, 0.12440251559019089, 0.0, 0.0, 0.0], [0.6535119414329529, 0.07573550939559937, 0.09732570499181747, 0.17342683672904968, 0.0, 0.0], [0.5222766995429993, 0.05827884376049042, 0.09920478612184525, 0.1702084094285965, 0.15003129839897156, 0.0], [0.4108838438987732, 0.047306086868047714, 0.07265683263540268, 0.10560749471187592, 0.10550011694431305, 0.25804561376571655]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9683831930160522, 0.031616728752851486, 0.0, 0.0, 0.0, 0.0], [0.896539568901062, 0.03887060284614563, 0.06458979845046997, 0.0, 0.0, 0.0], [0.8264952898025513, 0.03213466331362724, 0.051967162638902664, 0.08940286934375763, 0.0, 0.0], [0.7718175053596497, 0.030402861535549164, 0.04582742601633072, 0.07118464261293411, 0.08076757192611694, 0.0], [0.7292331457138062, 0.021699873730540276, 0.033074744045734406, 0.04720093309879303, 0.06474556028842926, 0.10404572635889053]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979567527770996, 0.0020432821474969387, 0.0, 0.0, 0.0, 0.0], [0.9552940726280212, 0.008025333285331726, 0.03668065741658211, 0.0, 0.0, 0.0], [0.9254711270332336, 0.00275557953864336, 0.002062988467514515, 0.06971034407615662, 0.0, 0.0], [0.8660575151443481, 0.003888372564688325, 0.0006785987643525004, 0.0006981458282098174, 0.12867732346057892, 0.0], [0.8455936908721924, 0.0037804015446454287, 0.0002534226223360747, 6.0270347603363916e-05, 0.00011820702638942748, 0.150193989276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9262455701828003, 0.07375437766313553, 0.0, 0.0, 0.0, 0.0], [0.7717147469520569, 0.1624203771352768, 0.06586486846208572, 0.0, 0.0, 0.0], [0.8167640566825867, 0.07807155698537827, 0.06324019283056259, 0.04192415252327919, 0.0, 0.0], [0.6867176294326782, 0.07755190879106522, 0.1005692109465599, 0.05955103039741516, 0.07561030983924866, 0.0], [0.6421176791191101, 0.11014875024557114, 0.07688140124082565, 0.05403309315443039, 0.1033359244465828, 0.013483130373060703]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9395952820777893, 0.06040477007627487, 0.0, 0.0, 0.0, 0.0], [0.2300449162721634, 0.6617392301559448, 0.10821589082479477, 0.0, 0.0, 0.0], [0.26702260971069336, 0.3607953190803528, 0.3249623775482178, 0.04721971973776817, 0.0, 0.0], [0.5952032804489136, 0.12269210070371628, 0.06302005052566528, 0.08916770666837692, 0.12991684675216675, 0.0], [0.1028466448187828, 0.029380010440945625, 0.01373895350843668, 0.04586036503314972, 0.7698498964309692, 0.03832418471574783]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9040982723236084, 0.0959017425775528, 0.0, 0.0, 0.0, 0.0], [0.357235312461853, 0.6274638175964355, 0.015300935134291649, 0.0, 0.0, 0.0], [0.5918000936508179, 0.27640387415885925, 0.1047603040933609, 0.027035744860768318, 0.0, 0.0], [0.7254436612129211, 0.04983079805970192, 0.01498265191912651, 0.17781229317188263, 0.03193056955933571, 0.0], [0.7612767219543457, 0.06158901005983353, 0.005942164920270443, 0.01642669551074505, 0.12677954137325287, 0.02798592671751976]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9947587847709656, 0.005241230130195618, 0.0, 0.0, 0.0, 0.0], [0.9632415771484375, 0.017816435545682907, 0.018942005932331085, 0.0, 0.0, 0.0], [0.9671075940132141, 0.008509604260325432, 0.00856224074959755, 0.015820497646927834, 0.0, 0.0], [0.9340998530387878, 0.011952383443713188, 0.020180154591798782, 0.026750771328806877, 0.007016891613602638, 0.0], [0.9587240219116211, 0.0046570925042033195, 0.003326763864606619, 0.006545269396156073, 0.010182411409914494, 0.016564475372433662]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769991040229797, 0.02300085686147213, 0.0, 0.0, 0.0, 0.0], [0.791760265827179, 0.1753326654434204, 0.032907143235206604, 0.0, 0.0, 0.0], [0.7949190735816956, 0.10531850159168243, 0.04021850973367691, 0.05954388156533241, 0.0, 0.0], [0.7097727060317993, 0.10552525520324707, 0.06597540527582169, 0.05765572935342789, 0.06107088923454285, 0.0], [0.7506603598594666, 0.026514418423175812, 0.02157600037753582, 0.034296635538339615, 0.08494450896978378, 0.08200804889202118]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.983751654624939, 0.01624833419919014, 0.0, 0.0, 0.0, 0.0], [0.5615487694740295, 0.08956838399171829, 0.3488827645778656, 0.0, 0.0, 0.0], [0.3292914927005768, 0.024114832282066345, 0.5428048968315125, 0.10378878563642502, 0.0, 0.0], [0.34330493211746216, 0.013086404651403427, 0.5121950507164001, 0.11146263778209686, 0.01995093934237957, 0.0], [0.4792822599411011, 0.017333481460809708, 0.11805278062820435, 0.06130271032452583, 0.20071901381015778, 0.12330969423055649]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9908847212791443, 0.009115317836403847, 0.0, 0.0, 0.0, 0.0], [0.5282679200172424, 0.3292284309864044, 0.14250363409519196, 0.0, 0.0, 0.0], [0.48788732290267944, 0.233686164021492, 0.17577986419200897, 0.10264668613672256, 0.0, 0.0], [0.31444936990737915, 0.18065084517002106, 0.16871292889118195, 0.09506543725728989, 0.24112139642238617, 0.0], [0.5168782472610474, 0.03589695692062378, 0.026187807321548462, 0.040397077798843384, 0.18791691958904266, 0.19272294640541077]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8750306367874146, 0.12496936321258545, 0.0, 0.0, 0.0, 0.0], [0.4550616145133972, 0.4900423586368561, 0.05489595979452133, 0.0, 0.0, 0.0], [0.29337263107299805, 0.5449898838996887, 0.09444298595190048, 0.06719449907541275, 0.0, 0.0], [0.4897097647190094, 0.2720980942249298, 0.06861936300992966, 0.14694873988628387, 0.02262401394546032, 0.0], [0.4729084074497223, 0.08103056997060776, 0.01605204865336418, 0.3067218065261841, 0.10120705515146255, 0.022080253809690475]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9630220532417297, 0.03697795420885086, 0.0, 0.0, 0.0, 0.0], [0.7557179927825928, 0.1643647700548172, 0.07991719990968704, 0.0, 0.0, 0.0], [0.6947709918022156, 0.08409848809242249, 0.06382585316896439, 0.15730471909046173, 0.0, 0.0], [0.5821163654327393, 0.03297785297036171, 0.07936536520719528, 0.1944136917591095, 0.11112680286169052, 0.0], [0.5974539518356323, 0.04261082038283348, 0.06919696182012558, 0.14563480019569397, 0.12481768429279327, 0.02028587833046913]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9957822561264038, 0.004217814654111862, 0.0, 0.0, 0.0, 0.0], [0.9312829971313477, 0.010560262948274612, 0.058156777173280716, 0.0, 0.0, 0.0], [0.8435326218605042, 0.015694990754127502, 0.045751187950372696, 0.09502114355564117, 0.0, 0.0], [0.7724104523658752, 0.011981194838881493, 0.035046011209487915, 0.038767579942941666, 0.14179480075836182, 0.0], [0.7642906904220581, 0.00986877828836441, 0.008122753351926804, 0.013314371928572655, 0.04824399948120117, 0.15615928173065186]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9701175093650818, 0.029882490634918213, 0.0, 0.0, 0.0, 0.0], [0.6563995480537415, 0.22506217658519745, 0.11853817850351334, 0.0, 0.0, 0.0], [0.6958061456680298, 0.1470184177160263, 0.07145984470844269, 0.08571554720401764, 0.0, 0.0], [0.6353272795677185, 0.13460662961006165, 0.030994117259979248, 0.05691640451550484, 0.1421555131673813, 0.0], [0.6779407262802124, 0.05365397781133652, 0.018006201833486557, 0.06284506618976593, 0.11038198322057724, 0.07717210799455643]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9822332859039307, 0.01776670478284359, 0.0, 0.0, 0.0, 0.0], [0.9037660956382751, 0.06541543453931808, 0.030818426981568336, 0.0, 0.0, 0.0], [0.8119187355041504, 0.036790236830711365, 0.06056087091565132, 0.09073004871606827, 0.0, 0.0], [0.40546342730522156, 0.10383892804384232, 0.10211260616779327, 0.3543427288532257, 0.03424236178398132, 0.0], [0.22824503481388092, 0.017278604209423065, 0.05055435746908188, 0.6015732288360596, 0.09411737322807312, 0.008231470361351967]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873148202896118, 0.012685154564678669, 0.0, 0.0, 0.0, 0.0], [0.354457825422287, 0.531760036945343, 0.11378218978643417, 0.0, 0.0, 0.0], [0.07823401689529419, 0.7221351265907288, 0.10936620831489563, 0.09026475995779037, 0.0, 0.0], [0.21968045830726624, 0.40484222769737244, 0.1235806941986084, 0.2001885175704956, 0.051708124577999115, 0.0], [0.36089763045310974, 0.1045902818441391, 0.06983798742294312, 0.29764822125434875, 0.13869896531105042, 0.028326835483312607]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9732161164283752, 0.026783881708979607, 0.0, 0.0, 0.0, 0.0], [0.9167553782463074, 0.06145283579826355, 0.02179183065891266, 0.0, 0.0, 0.0], [0.8543079495429993, 0.08049621433019638, 0.03033490665256977, 0.03486097976565361, 0.0, 0.0], [0.8919220566749573, 0.04280766472220421, 0.022044911980628967, 0.023470569401979446, 0.01975482702255249, 0.0], [0.8116769194602966, 0.03413527458906174, 0.03567648306488991, 0.04748578742146492, 0.025397121906280518, 0.045628443360328674]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9502761363983154, 0.04972386732697487, 0.0, 0.0, 0.0, 0.0], [0.7637453675270081, 0.20073619484901428, 0.03551839664578438, 0.0, 0.0, 0.0], [0.6279090046882629, 0.037681419402360916, 0.19945412874221802, 0.13495545089244843, 0.0, 0.0], [0.6397066712379456, 0.027007192373275757, 0.09081999957561493, 0.20653821527957916, 0.03592786192893982, 0.0], [0.45594266057014465, 0.02164124697446823, 0.12939591705799103, 0.21800900995731354, 0.10379799455404282, 0.07121305912733078]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9498405456542969, 0.05015944689512253, 0.0, 0.0, 0.0, 0.0], [0.868872344493866, 0.08722198009490967, 0.04390567168593407, 0.0, 0.0, 0.0], [0.6937950849533081, 0.06359203159809113, 0.09179060161113739, 0.1508222371339798, 0.0, 0.0], [0.7266592383384705, 0.04389891028404236, 0.046839818358421326, 0.09851827472448349, 0.08408384770154953, 0.0], [0.7848992943763733, 0.037147846072912216, 0.012907843105494976, 0.010539384558796883, 0.12079204618930817, 0.03371358662843704]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891054034233093, 0.01089461799710989, 0.0, 0.0, 0.0, 0.0], [0.8929510116577148, 0.08700131624937057, 0.020047729834914207, 0.0, 0.0, 0.0], [0.7891115546226501, 0.09797312319278717, 0.0863322764635086, 0.026583032682538033, 0.0, 0.0], [0.8850637674331665, 0.0364501029253006, 0.053954314440488815, 0.012377242557704449, 0.01215458381921053, 0.0], [0.6861329078674316, 0.057203810662031174, 0.0116362813860178, 0.02166050672531128, 0.17488020658493042, 0.048486314713954926]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9396193027496338, 0.06038068234920502, 0.0, 0.0, 0.0, 0.0], [0.7851802110671997, 0.19751375913619995, 0.01730601117014885, 0.0, 0.0, 0.0], [0.7660512328147888, 0.15444660186767578, 0.0318828783929348, 0.04761931300163269, 0.0, 0.0], [0.7035237550735474, 0.05171409994363785, 0.0776095986366272, 0.15339010953903198, 0.013762442395091057, 0.0], [0.7121893763542175, 0.04994220659136772, 0.03772540017962456, 0.08649095147848129, 0.06541402637958527, 0.04823804274201393]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.974072277545929, 0.025927739217877388, 0.0, 0.0, 0.0, 0.0], [0.7925394773483276, 0.011715609580278397, 0.19574493169784546, 0.0, 0.0, 0.0], [0.5106772184371948, 0.007296781521290541, 0.03962003067135811, 0.44240593910217285, 0.0, 0.0], [0.5862484574317932, 0.01209966279566288, 0.024585144594311714, 0.06737807393074036, 0.30968865752220154, 0.0], [0.3019636273384094, 0.007724008988589048, 0.011518117040395737, 0.04694722592830658, 0.2214670479297638, 0.41037997603416443]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9744554162025452, 0.02554461918771267, 0.0, 0.0, 0.0, 0.0], [0.9769195318222046, 0.015048524364829063, 0.00803192425519228, 0.0, 0.0, 0.0], [0.9060619473457336, 0.025875424966216087, 0.02595478855073452, 0.04210774973034859, 0.0, 0.0], [0.9400084614753723, 0.00555663276463747, 0.005828281864523888, 0.031757764518260956, 0.016849050298333168, 0.0], [0.9105738401412964, 0.0019752182997763157, 0.008646713569760323, 0.013360859826207161, 0.03543969988822937, 0.030003685504198074]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9791666865348816, 0.02083336003124714, 0.0, 0.0, 0.0, 0.0], [0.8444863557815552, 0.13507835566997528, 0.020435357466340065, 0.0, 0.0, 0.0], [0.7903085947036743, 0.14559166133403778, 0.037530042231082916, 0.026569712907075882, 0.0, 0.0], [0.7298934459686279, 0.05649632588028908, 0.03273547440767288, 0.10400423407554626, 0.07687054574489594, 0.0], [0.5684199929237366, 0.04388810321688652, 0.026293331757187843, 0.08117101341485977, 0.2431478202342987, 0.037079744040966034]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9499868154525757, 0.050013184547424316, 0.0, 0.0, 0.0, 0.0], [0.9336171746253967, 0.05848868191242218, 0.007894255220890045, 0.0, 0.0, 0.0], [0.7897834181785583, 0.11071813106536865, 0.05360180512070656, 0.045896656811237335, 0.0, 0.0], [0.8859304785728455, 0.05752970278263092, 0.013743232935667038, 0.0033877352252602577, 0.03940894827246666, 0.0], [0.9337607622146606, 0.026470666751265526, 0.004523405339568853, 0.0061904969625175, 0.01413296815007925, 0.014921738766133785]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.0, 1.8521648659231005e-09, 0.0, 0.0, 0.0, 0.0], [6.6758143475453835e-06, 0.9999804496765137, 1.284119480260415e-05, 0.0, 0.0, 0.0], [2.219406880499264e-08, 2.668508969350114e-09, 0.9999971389770508, 2.813725814121426e-06, 0.0, 0.0], [1.0145217856916133e-06, 4.4640319885047575e-08, 0.0003535518771968782, 0.9993677735328674, 0.00027762516401708126, 0.0], [9.436730286083161e-10, 1.3820468207359493e-11, 5.017710691390675e-10, 2.9652009736480522e-09, 0.9999971389770508, 2.864445150407846e-06]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9948632121086121, 0.00513678602874279, 0.0, 0.0, 0.0, 0.0], [0.9274216890335083, 0.018323812633752823, 0.05425459146499634, 0.0, 0.0, 0.0], [0.9678993225097656, 0.0041434201411902905, 0.004314461722970009, 0.023642871528863907, 0.0, 0.0], [0.899907648563385, 0.0014671608805656433, 0.00029133600764907897, 0.002585014561191201, 0.09574881941080093, 0.0], [0.9386116862297058, 0.00022248241293709725, 0.000614665390457958, 0.0015495705883949995, 0.030689310282468796, 0.02831241302192211]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9999959468841553, 4.042735781695228e-06, 0.0, 0.0, 0.0, 0.0], [0.9982761144638062, 3.2613615985610522e-06, 0.001720665255561471, 0.0, 0.0, 0.0], [0.9998809099197388, 5.328835683826583e-08, 6.376233727678482e-07, 0.00011847116547869518, 0.0, 0.0], [0.9996154308319092, 3.4731792197817413e-07, 3.892067468314053e-08, 4.468433303372876e-07, 0.00038369561661966145, 0.0], [0.9994840621948242, 1.6550142589721872e-08, 2.871539450666205e-08, 1.0638284493325045e-06, 0.00021266516705509275, 0.0003021192387677729]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9514135718345642, 0.048586443066596985, 0.0, 0.0, 0.0, 0.0], [0.5749930143356323, 0.39028263092041016, 0.03472428396344185, 0.0, 0.0, 0.0], [0.744231641292572, 0.1752413809299469, 0.07564764469861984, 0.004879307933151722, 0.0, 0.0], [0.5232048034667969, 0.09429354965686798, 0.11381909251213074, 0.199794739484787, 0.06888782978057861, 0.0], [0.4747248888015747, 0.05636602267622948, 0.045303650200366974, 0.06967318058013916, 0.3098025918006897, 0.044129688292741776]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8734647631645203, 0.12653520703315735, 0.0, 0.0, 0.0, 0.0], [0.6097909212112427, 0.3541729748249054, 0.03603604808449745, 0.0, 0.0, 0.0], [0.4598419666290283, 0.3869785964488983, 0.09960105270147324, 0.05357832461595535, 0.0, 0.0], [0.5722200274467468, 0.23636257648468018, 0.08344569057226181, 0.06921947747468948, 0.038752224296331406, 0.0], [0.514358401298523, 0.16722990572452545, 0.09019368141889572, 0.0765446200966835, 0.10578057914972305, 0.045892804861068726]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981228768825531, 0.018771234899759293, 0.0, 0.0, 0.0, 0.0], [0.6142938137054443, 0.35039809346199036, 0.03530815243721008, 0.0, 0.0, 0.0], [0.5770687460899353, 0.32858437299728394, 0.05508258566260338, 0.03926431015133858, 0.0, 0.0], [0.17188090085983276, 0.011042523197829723, 0.05457884445786476, 0.7326592803001404, 0.02983849309384823, 0.0], [0.3783024251461029, 0.017070062458515167, 0.021754097193479538, 0.4409683644771576, 0.06093815341591835, 0.08096696436405182]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9923112392425537, 0.007688735146075487, 0.0, 0.0, 0.0, 0.0], [0.9498787522315979, 0.016709759831428528, 0.033411506563425064, 0.0, 0.0, 0.0], [0.9961295127868652, 0.0008787286351434886, 0.0006868178606964648, 0.002304842695593834, 0.0, 0.0], [0.9935757517814636, 0.003263507504016161, 0.0009993839776143432, 0.0002793238090816885, 0.001882061013020575, 0.0], [0.9907532930374146, 0.00021344359265640378, 0.0004595248610712588, 0.0007905631209723651, 0.004424731712788343, 0.003358363639563322]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9647740125656128, 0.03522596135735512, 0.0, 0.0, 0.0, 0.0], [0.8194130659103394, 0.1365436464548111, 0.044043250381946564, 0.0, 0.0, 0.0], [0.7584242820739746, 0.006878956686705351, 0.20653414726257324, 0.028162650763988495, 0.0, 0.0], [0.5298121571540833, 0.002678812248632312, 0.07857993245124817, 0.3598378598690033, 0.029091132804751396, 0.0], [0.754442572593689, 0.0003678239299915731, 0.001971345627680421, 0.003240057732909918, 0.19423308968544006, 0.04574507102370262]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9749132394790649, 0.025086797773838043, 0.0, 0.0, 0.0, 0.0], [0.9306469559669495, 0.057056792080402374, 0.012296278029680252, 0.0, 0.0, 0.0], [0.9305251836776733, 0.05277099460363388, 0.011119411326944828, 0.005584418307989836, 0.0, 0.0], [0.8863323926925659, 0.012924155220389366, 0.017724651843309402, 0.06150183826684952, 0.021517043933272362, 0.0], [0.7916852831840515, 0.015036062337458134, 0.031747858971357346, 0.033922016620635986, 0.03707969933748245, 0.09052912145853043]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.960849940776825, 0.039150021970272064, 0.0, 0.0, 0.0, 0.0], [0.9121273159980774, 0.022576453164219856, 0.06529619544744492, 0.0, 0.0, 0.0], [0.9364106059074402, 0.01558446604758501, 0.024544961750507355, 0.023459969088435173, 0.0, 0.0], [0.9454619288444519, 0.006762297358363867, 0.022026164457201958, 0.009137828834354877, 0.0166118573397398, 0.0], [0.8346167802810669, 0.0018816972151398659, 0.005609032232314348, 0.01887362077832222, 0.1244916245341301, 0.014527121558785439]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964227080345154, 0.003577286144718528, 0.0, 0.0, 0.0, 0.0], [0.9713928699493408, 0.024453043937683105, 0.004154064226895571, 0.0, 0.0, 0.0], [0.9735792279243469, 0.01900341734290123, 0.0036644143983721733, 0.003752920078113675, 0.0, 0.0], [0.9586310982704163, 0.007116200402379036, 0.009218422695994377, 0.02272571064531803, 0.0023084881249815226, 0.0], [0.973607063293457, 0.008490582928061485, 0.0032512503676116467, 0.0036064486484974623, 0.004877470899373293, 0.006167230196297169]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97598797082901, 0.024012047797441483, 0.0, 0.0, 0.0, 0.0], [0.946063756942749, 0.04211379587650299, 0.011822430416941643, 0.0, 0.0, 0.0], [0.8446813225746155, 0.04293113946914673, 0.05218193307518959, 0.0602056086063385, 0.0, 0.0], [0.9378372430801392, 0.03354857116937637, 0.008826462551951408, 0.0028792363591492176, 0.016908491030335426, 0.0], [0.8124933838844299, 0.026967575773596764, 0.05999194458127022, 0.03445727750658989, 0.011011868715286255, 0.05507809296250343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9001206755638123, 0.09987929463386536, 0.0, 0.0, 0.0, 0.0], [0.6271925568580627, 0.07988723367452621, 0.2929201126098633, 0.0, 0.0, 0.0], [0.7624078989028931, 0.02734428085386753, 0.03867945075035095, 0.17156831920146942, 0.0, 0.0], [0.7995962500572205, 0.014336277730762959, 0.014375667087733746, 0.025438543409109116, 0.1462532877922058, 0.0], [0.7851975560188293, 0.04204043000936508, 0.025253599509596825, 0.02908393368124962, 0.029306255280971527, 0.08911828696727753]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954467415809631, 0.004553207661956549, 0.0, 0.0, 0.0, 0.0], [0.9356004595756531, 0.04476727917790413, 0.019632209092378616, 0.0, 0.0, 0.0], [0.5605551600456238, 0.09862001240253448, 0.2998324930667877, 0.04099231958389282, 0.0, 0.0], [0.589370608329773, 0.1100098192691803, 0.08033646643161774, 0.16754046082496643, 0.0527426153421402, 0.0], [0.223059743642807, 0.056807976216077805, 0.05467941239476204, 0.24733929336071014, 0.3111240565776825, 0.10698945075273514]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9301448464393616, 0.06985516101121902, 0.0, 0.0, 0.0, 0.0], [0.8936478495597839, 0.08535714447498322, 0.020995058119297028, 0.0, 0.0, 0.0], [0.8404536843299866, 0.10619213432073593, 0.023636789992451668, 0.029717300087213516, 0.0, 0.0], [0.8927381634712219, 0.02478480525314808, 0.008319051936268806, 0.05165467783808708, 0.022503303363919258, 0.0], [0.8646609783172607, 0.009503169916570187, 0.00243298034183681, 0.04796737805008888, 0.042732127010822296, 0.032703299075365067]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859625697135925, 0.014037402346730232, 0.0, 0.0, 0.0, 0.0], [0.9702038764953613, 0.01680702343583107, 0.012989096343517303, 0.0, 0.0, 0.0], [0.9524770379066467, 0.016064444556832314, 0.013456214219331741, 0.018002301454544067, 0.0, 0.0], [0.9332928657531738, 0.018971974030137062, 0.020146867260336876, 0.017023736611008644, 0.010564574971795082, 0.0], [0.9113593697547913, 0.012528610415756702, 0.022096145898103714, 0.017518552020192146, 0.01851789653301239, 0.01797933503985405]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9681769013404846, 0.03182312101125717, 0.0, 0.0, 0.0, 0.0], [0.9096419215202332, 0.07916687428951263, 0.011191246099770069, 0.0, 0.0, 0.0], [0.8379933834075928, 0.13078251481056213, 0.012140998616814613, 0.019083019345998764, 0.0, 0.0], [0.9116523265838623, 0.05451972037553787, 0.00949938129633665, 0.007465865928679705, 0.016862763091921806, 0.0], [0.8510287404060364, 0.07338211685419083, 0.008022502064704895, 0.009083151817321777, 0.04261011257767677, 0.015873325988650322]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9799023866653442, 0.02009769156575203, 0.0, 0.0, 0.0, 0.0], [0.9558742642402649, 0.029063263908028603, 0.015062462538480759, 0.0, 0.0, 0.0], [0.7943130731582642, 0.06074107438325882, 0.0690767914056778, 0.0758691281080246, 0.0, 0.0], [0.549432098865509, 0.031547121703624725, 0.054820187389850616, 0.0578807033598423, 0.306319922208786, 0.0], [0.645397961139679, 0.010770927183330059, 0.017528077587485313, 0.02157982438802719, 0.24958311021327972, 0.05514014512300491]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9506808519363403, 0.049319129437208176, 0.0, 0.0, 0.0, 0.0], [0.8553208708763123, 0.09256317466497421, 0.05211597681045532, 0.0, 0.0, 0.0], [0.8508525490760803, 0.04734596610069275, 0.04417724534869194, 0.057624202221632004, 0.0, 0.0], [0.7697128653526306, 0.027885887771844864, 0.031017253175377846, 0.06842502951622009, 0.10295900702476501, 0.0], [0.7931906580924988, 0.040521834045648575, 0.029241926968097687, 0.04478121176362038, 0.04894692823290825, 0.04331746697425842]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9770310521125793, 0.022968946024775505, 0.0, 0.0, 0.0, 0.0], [0.9429818391799927, 0.017321474850177765, 0.03969673812389374, 0.0, 0.0, 0.0], [0.9144347906112671, 0.008583571761846542, 0.013035777024924755, 0.06394589692354202, 0.0, 0.0], [0.9222431182861328, 0.003644028678536415, 0.003740261774510145, 0.010410326533019543, 0.05996226519346237, 0.0], [0.9198879599571228, 0.0030822642147541046, 0.0034827410709112883, 0.0042068008333444595, 0.021254301071166992, 0.04808594286441803]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9774582386016846, 0.02254181168973446, 0.0, 0.0, 0.0, 0.0], [0.8929324150085449, 0.07475479692220688, 0.032312821596860886, 0.0, 0.0, 0.0], [0.8423513174057007, 0.05980277061462402, 0.037400756031274796, 0.060445044189691544, 0.0, 0.0], [0.7674633264541626, 0.035363439470529556, 0.04215509817004204, 0.06658629328012466, 0.08843192458152771, 0.0], [0.6182613372802734, 0.016110623255372047, 0.020167628303170204, 0.03868889808654785, 0.2314697951078415, 0.0753016546368599]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9634856581687927, 0.036514345556497574, 0.0, 0.0, 0.0, 0.0], [0.43639448285102844, 0.5226364731788635, 0.040969014167785645, 0.0, 0.0, 0.0], [0.36086124181747437, 0.35129690170288086, 0.2655107080936432, 0.02233118563890457, 0.0, 0.0], [0.39429229497909546, 0.021704625338315964, 0.07794322818517685, 0.3716890215873718, 0.13437077403068542, 0.0], [0.6310721635818481, 0.016983957961201668, 0.025942014530301094, 0.08615921437740326, 0.21831963956356049, 0.021522965282201767]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9988250136375427, 0.0011750265257433057, 0.0, 0.0, 0.0, 0.0], [0.9944871068000793, 0.0004826387739740312, 0.005030233412981033, 0.0, 0.0, 0.0], [0.9981209635734558, 2.7051630240748636e-05, 0.00011307397653581575, 0.0017389433924108744, 0.0, 0.0], [0.9982239603996277, 6.836584361735731e-05, 0.00010199935059063137, 6.028370262356475e-05, 0.001545313629321754, 0.0], [0.9982888102531433, 1.0552178082434693e-06, 3.278081203461625e-05, 0.0001303895260207355, 0.0006605856469832361, 0.0008863675757311285]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9936710596084595, 0.006328933872282505, 0.0, 0.0, 0.0, 0.0], [0.9727688431739807, 0.0018561349716037512, 0.025375083088874817, 0.0, 0.0, 0.0], [0.9724301099777222, 0.00195861142128706, 0.011192452162504196, 0.014418844133615494, 0.0, 0.0], [0.9782041311264038, 0.0009589171386323869, 0.001870649284683168, 0.006326562725007534, 0.01263973768800497, 0.0], [0.9592596888542175, 0.002455513458698988, 0.0016124167013913393, 0.005019667558372021, 0.006687135435640812, 0.024965722113847733]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9629000425338745, 0.03709990158677101, 0.0, 0.0, 0.0, 0.0], [0.36802080273628235, 0.6152244210243225, 0.016754813492298126, 0.0, 0.0, 0.0], [0.31735214591026306, 0.6140003800392151, 0.053751468658447266, 0.014896074309945107, 0.0, 0.0], [0.4898741543292999, 0.21071387827396393, 0.04693012312054634, 0.20700415968894958, 0.04547771066427231, 0.0], [0.48774242401123047, 0.1769523322582245, 0.06915220618247986, 0.09849317371845245, 0.12091425806283951, 0.04674568027257919]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9794419407844543, 0.02055807039141655, 0.0, 0.0, 0.0, 0.0], [0.6677890419960022, 0.31032490730285645, 0.021886074915528297, 0.0, 0.0, 0.0], [0.711875855922699, 0.11108537018299103, 0.14187364280223846, 0.03516502305865288, 0.0, 0.0], [0.4501466155052185, 0.04036048427224159, 0.04045815393328667, 0.3885694742202759, 0.0804651752114296, 0.0], [0.49346205592155457, 0.013696958310902119, 0.00812679436057806, 0.13074517250061035, 0.30861401557922363, 0.04535504803061485]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9846054315567017, 0.015394601970911026, 0.0, 0.0, 0.0, 0.0], [0.9806739091873169, 0.007713806349784136, 0.011612341739237309, 0.0, 0.0, 0.0], [0.9326630234718323, 0.01957845874130726, 0.024103542789816856, 0.02365495264530182, 0.0, 0.0], [0.9422018527984619, 0.0009538960293866694, 0.0010898001492023468, 0.0031933635473251343, 0.05256103351712227, 0.0], [0.9352929592132568, 0.0010279365815222263, 0.004444439895451069, 0.0016371352830901742, 0.010590966790914536, 0.047006525099277496]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985783100128174, 0.0014216770650818944, 0.0, 0.0, 0.0, 0.0], [0.9893348813056946, 0.0011178908171132207, 0.009547143243253231, 0.0, 0.0, 0.0], [0.9979978203773499, 7.997050124686211e-05, 0.00013218825915828347, 0.0017900299280881882, 0.0, 0.0], [0.9986976385116577, 4.1044117097044364e-05, 3.868332441925304e-06, 2.3676169803366065e-05, 0.0012337109073996544, 0.0], [0.9971562623977661, 1.8522179743740708e-05, 1.8826593759513344e-06, 2.7900150598725304e-05, 0.0006533485138788819, 0.002141993958503008]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9768233299255371, 0.023176709190011024, 0.0, 0.0, 0.0, 0.0], [0.919467568397522, 0.050882015377283096, 0.029650306329131126, 0.0, 0.0, 0.0], [0.8474555015563965, 0.06100171059370041, 0.04372366517782211, 0.047819141298532486, 0.0, 0.0], [0.8011622428894043, 0.041867081075906754, 0.04375801607966423, 0.041894808411598206, 0.07131782919168472, 0.0], [0.8031870126724243, 0.024504918605089188, 0.01732351817190647, 0.0474439337849617, 0.06109936535358429, 0.04644131660461426]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9829428195953369, 0.01705716922879219, 0.0, 0.0, 0.0, 0.0], [0.8863736391067505, 0.09492656588554382, 0.018699748441576958, 0.0, 0.0, 0.0], [0.9231085777282715, 0.03696347773075104, 0.03219832852482796, 0.007729670964181423, 0.0, 0.0], [0.9068530201911926, 0.01604657620191574, 0.014310453087091446, 0.04543779417872429, 0.01735224947333336, 0.0], [0.6555968523025513, 0.05091030150651932, 0.028384873643517494, 0.125655397772789, 0.1054687574505806, 0.03398393467068672]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9502318501472473, 0.04976809769868851, 0.0, 0.0, 0.0, 0.0], [0.8829867243766785, 0.1000962033867836, 0.01691715233027935, 0.0, 0.0, 0.0], [0.8057456016540527, 0.14463560283184052, 0.03018922172486782, 0.01942945271730423, 0.0, 0.0], [0.8706232905387878, 0.03244058042764664, 0.02695159800350666, 0.04410297051072121, 0.025881517678499222, 0.0], [0.6883644461631775, 0.00968148186802864, 0.016449354588985443, 0.09871124476194382, 0.08971203118562698, 0.09708139300346375]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9792683124542236, 0.020731747150421143, 0.0, 0.0, 0.0, 0.0], [0.9523284435272217, 0.025933869183063507, 0.02173773944377899, 0.0, 0.0, 0.0], [0.9144353866577148, 0.017671285197138786, 0.02235853485763073, 0.045534808188676834, 0.0, 0.0], [0.9448293447494507, 0.006467591971158981, 0.006386054679751396, 0.03263082355260849, 0.009686139412224293, 0.0], [0.9347907304763794, 0.007862498052418232, 0.007788168732076883, 0.02143275924026966, 0.008491129614412785, 0.0196347888559103]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9833701848983765, 0.01662975549697876, 0.0, 0.0, 0.0, 0.0], [0.9631112813949585, 0.009229970164597034, 0.02765871398150921, 0.0, 0.0, 0.0], [0.9706628322601318, 0.004149409011006355, 0.006813114508986473, 0.01837460696697235, 0.0, 0.0], [0.987951934337616, 0.0021658889017999172, 0.0003490102826617658, 0.001583811710588634, 0.007949435152113438, 0.0], [0.945794939994812, 0.014583585783839226, 0.00036529666977003217, 0.0009569558897055686, 0.01362166740000248, 0.024677546694874763]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9878059029579163, 0.012194057926535606, 0.0, 0.0, 0.0, 0.0], [0.8710366487503052, 0.0944816991686821, 0.03448163717985153, 0.0, 0.0, 0.0], [0.6309780478477478, 0.11090393364429474, 0.1923023909330368, 0.06581564992666245, 0.0, 0.0], [0.5360495448112488, 0.04618944227695465, 0.13605298101902008, 0.2645546793937683, 0.01715332642197609, 0.0], [0.8287513852119446, 0.02373281493782997, 0.02008042484521866, 0.07245299965143204, 0.0304313562810421, 0.02455105073750019]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8995685577392578, 0.10043143481016159, 0.0, 0.0, 0.0, 0.0], [0.270342618227005, 0.6504339575767517, 0.07922345399856567, 0.0, 0.0, 0.0], [0.20541740953922272, 0.5892512798309326, 0.180857852101326, 0.02447350136935711, 0.0, 0.0], [0.5573872327804565, 0.17741312086582184, 0.0880676656961441, 0.09881818294525146, 0.07831382751464844, 0.0], [0.5922901034355164, 0.08700644969940186, 0.056432779878377914, 0.05685899406671524, 0.1218155100941658, 0.08559606969356537]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9316376447677612, 0.068362295627594, 0.0, 0.0, 0.0, 0.0], [0.9572947025299072, 0.02624349854886532, 0.016461782157421112, 0.0, 0.0, 0.0], [0.9880544543266296, 0.004273331258445978, 0.0029545812867581844, 0.004717642907053232, 0.0, 0.0], [0.99403977394104, 0.0009413447696715593, 0.00047398178139701486, 0.00011646964412648231, 0.004428428132086992, 0.0], [0.9806035161018372, 2.5468958483543247e-05, 0.00016239396063610911, 0.0001476412871852517, 0.001344241201877594, 0.017716852948069572]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.993178129196167, 0.006821870803833008, 0.0, 0.0, 0.0, 0.0], [0.9756524562835693, 0.013184132054448128, 0.011163455434143543, 0.0, 0.0, 0.0], [0.9418967962265015, 0.004721763078123331, 0.0023818148765712976, 0.0509997121989727, 0.0, 0.0], [0.9905040860176086, 0.0022848148364573717, 6.19848578935489e-05, 0.0005984479794278741, 0.006550685502588749, 0.0], [0.9697661399841309, 0.0008878838270902634, 0.0002346673863939941, 0.0017040770035237074, 0.004128350410610437, 0.02327890321612358]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9716230630874634, 0.0283768679946661, 0.0, 0.0, 0.0, 0.0], [0.9223619699478149, 0.028907284140586853, 0.048730771988630295, 0.0, 0.0, 0.0], [0.8426315784454346, 0.02387218549847603, 0.04748142883181572, 0.0860147550702095, 0.0, 0.0], [0.8521116375923157, 0.020744290202856064, 0.044946324080228806, 0.05765015631914139, 0.024547478184103966, 0.0], [0.8800724744796753, 0.02244853600859642, 0.01823573186993599, 0.01925482228398323, 0.015854286029934883, 0.04413414001464844]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9412722587585449, 0.05872778967022896, 0.0, 0.0, 0.0, 0.0], [0.9163140654563904, 0.05759185180068016, 0.026094017550349236, 0.0, 0.0, 0.0], [0.8392423987388611, 0.05769050121307373, 0.013829062692821026, 0.08923796564340591, 0.0, 0.0], [0.8987160921096802, 0.013477860949933529, 0.0003456451231613755, 0.0032987555023282766, 0.08416163921356201, 0.0], [0.8701690435409546, 0.0027008659671992064, 0.0014349978882819414, 0.0056661684066057205, 0.08874314278364182, 0.031285740435123444]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9656725525856018, 0.03432748094201088, 0.0, 0.0, 0.0, 0.0], [0.9178615212440491, 0.062257926911115646, 0.019880497828125954, 0.0, 0.0, 0.0], [0.8233146667480469, 0.06282396614551544, 0.03670436143875122, 0.07715708762407303, 0.0, 0.0], [0.8501746654510498, 0.03816923871636391, 0.031964968889951706, 0.05160145089030266, 0.028089623898267746, 0.0], [0.6572403907775879, 0.0587739571928978, 0.04336011782288551, 0.09013210982084274, 0.08146586269140244, 0.06902758777141571]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9162064790725708, 0.083793506026268, 0.0, 0.0, 0.0, 0.0], [0.9451773762702942, 0.040992818772792816, 0.013829846866428852, 0.0, 0.0, 0.0], [0.8928354978561401, 0.05368676036596298, 0.01759696751832962, 0.035880785435438156, 0.0, 0.0], [0.8337051868438721, 0.04799617826938629, 0.033513251692056656, 0.0468086414039135, 0.03797678276896477, 0.0], [0.8167194724082947, 0.06337127834558487, 0.013286269269883633, 0.020469754934310913, 0.025292271748185158, 0.06086096167564392]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9525133371353149, 0.04748661443591118, 0.0, 0.0, 0.0, 0.0], [0.30198633670806885, 0.6520944833755493, 0.045919183641672134, 0.0, 0.0, 0.0], [0.2855835556983948, 0.5569521188735962, 0.14447391033172607, 0.012990497052669525, 0.0, 0.0], [0.8438040018081665, 0.03225131705403328, 0.03954288363456726, 0.06848171353340149, 0.01592012494802475, 0.0], [0.6664936542510986, 0.06095915287733078, 0.04064338654279709, 0.06804481148719788, 0.09186346083879471, 0.07199545204639435]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9682656526565552, 0.031734418123960495, 0.0, 0.0, 0.0, 0.0], [0.7385215759277344, 0.22856871783733368, 0.03290969505906105, 0.0, 0.0, 0.0], [0.5946669578552246, 0.23033174872398376, 0.1486765593290329, 0.02632465586066246, 0.0, 0.0], [0.6339261531829834, 0.058130085468292236, 0.09654293209314346, 0.14291933178901672, 0.06848142296075821, 0.0], [0.40375757217407227, 0.08945360779762268, 0.07635073363780975, 0.255870521068573, 0.14330360293388367, 0.031263869255781174]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869793653488159, 0.013020600192248821, 0.0, 0.0, 0.0, 0.0], [0.863138735294342, 0.11056642234325409, 0.026294825598597527, 0.0, 0.0, 0.0], [0.9488077759742737, 0.02861507050693035, 0.006535581778734922, 0.016041584312915802, 0.0, 0.0], [0.9672170877456665, 0.006604989990592003, 0.0004517161869443953, 0.004844421520829201, 0.020881719887256622, 0.0], [0.9354623556137085, 0.0204780176281929, 0.001170022296719253, 0.007056924514472485, 0.016318105161190033, 0.01951460912823677]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9846673011779785, 0.01533269602805376, 0.0, 0.0, 0.0, 0.0], [0.9052745699882507, 0.08373622596263885, 0.010989241302013397, 0.0, 0.0, 0.0], [0.814594030380249, 0.042837489396333694, 0.10568274557590485, 0.0368858203291893, 0.0, 0.0], [0.23519699275493622, 0.012018428184092045, 0.05280094966292381, 0.6516199111938477, 0.048363760113716125, 0.0], [0.31818681955337524, 0.01863236539065838, 0.03948163613677025, 0.3755547106266022, 0.20787166059017181, 0.040272727608680725]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9811733365058899, 0.018826739862561226, 0.0, 0.0, 0.0, 0.0], [0.8618937730789185, 0.0647917315363884, 0.07331447303295135, 0.0, 0.0, 0.0], [0.7664536833763123, 0.07330439984798431, 0.10353528708219528, 0.05670657008886337, 0.0, 0.0], [0.8128500580787659, 0.03215484321117401, 0.05900546908378601, 0.05416510999202728, 0.04182443022727966, 0.0], [0.8687862157821655, 0.02698776312172413, 0.02046988718211651, 0.01629730872809887, 0.032183703035116196, 0.035275109112262726]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9264581799507141, 0.07354183495044708, 0.0, 0.0, 0.0, 0.0], [0.8403534293174744, 0.06373776495456696, 0.0959087386727333, 0.0, 0.0, 0.0], [0.7330995202064514, 0.06451123207807541, 0.10380081832408905, 0.09858846664428711, 0.0, 0.0], [0.9143611788749695, 0.008257807232439518, 0.007320408709347248, 0.01796630211174488, 0.05209442973136902, 0.0], [0.8971914052963257, 0.008555497974157333, 0.007019439246505499, 0.014860527589917183, 0.03399759903550148, 0.03837534412741661]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9180346727371216, 0.08196533471345901, 0.0, 0.0, 0.0, 0.0], [0.8328660726547241, 0.12199065834283829, 0.045143257826566696, 0.0, 0.0, 0.0], [0.7994153499603271, 0.08744156360626221, 0.03605782240629196, 0.07708528637886047, 0.0, 0.0], [0.8809850215911865, 0.02074967324733734, 0.020554570481181145, 0.01712082512676716, 0.060589876025915146, 0.0], [0.7453039288520813, 0.04433402791619301, 0.022549191489815712, 0.033152736723423004, 0.03357040509581566, 0.12108971923589706]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9867060780525208, 0.013293899595737457, 0.0, 0.0, 0.0, 0.0], [0.982177734375, 0.012414142489433289, 0.0054081035777926445, 0.0, 0.0, 0.0], [0.96304851770401, 0.015290786512196064, 0.010345697402954102, 0.011314956471323967, 0.0, 0.0], [0.9213568568229675, 0.014132496900856495, 0.017639189958572388, 0.016567690297961235, 0.030303703621029854, 0.0], [0.9373330473899841, 0.009064311161637306, 0.007548339664936066, 0.006576436571776867, 0.011827593669295311, 0.027650361880660057]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9951004385948181, 0.00489962799474597, 0.0, 0.0, 0.0, 0.0], [0.9476007223129272, 0.041407953947782516, 0.010991264134645462, 0.0, 0.0, 0.0], [0.9142175316810608, 0.023523805662989616, 0.03914501518011093, 0.02311362884938717, 0.0, 0.0], [0.9534734487533569, 0.008932976052165031, 0.015272831544280052, 0.007908296771347523, 0.01441233605146408, 0.0], [0.9427104592323303, 0.008233043365180492, 0.0046509732492268085, 0.004178094677627087, 0.0054635051637887955, 0.03476400673389435]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9543375372886658, 0.04566241055727005, 0.0, 0.0, 0.0, 0.0], [0.9696040749549866, 0.019547609612345695, 0.010848305188119411, 0.0, 0.0, 0.0], [0.9710449576377869, 0.012425409629940987, 0.008068887516856194, 0.008460711687803268, 0.0, 0.0], [0.9726192951202393, 0.002697661053389311, 0.0004483141237869859, 0.0013814771082252264, 0.022853175178170204, 0.0], [0.9675466418266296, 0.009613418020308018, 0.0032030276488512754, 0.004248825367540121, 0.0074422359466552734, 0.007945875637233257]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9887008666992188, 0.011299148201942444, 0.0, 0.0, 0.0, 0.0], [0.9382631182670593, 0.04204246401786804, 0.019694389775395393, 0.0, 0.0, 0.0], [0.8351995944976807, 0.034878626465797424, 0.051344748586416245, 0.07857709378004074, 0.0, 0.0], [0.9042678475379944, 0.010541587136685848, 0.016426727175712585, 0.02592189982533455, 0.04284190386533737, 0.0], [0.8913140296936035, 0.008912663906812668, 0.005010687280446291, 0.00817559752613306, 0.013514691032469273, 0.07307220995426178]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8693915009498596, 0.13060854375362396, 0.0, 0.0, 0.0, 0.0], [0.3507999777793884, 0.6063508987426758, 0.0428491048514843, 0.0, 0.0, 0.0], [0.35475727915763855, 0.3502015769481659, 0.24722374975681305, 0.04781733825802803, 0.0, 0.0], [0.35370829701423645, 0.03527732938528061, 0.09567102789878845, 0.4497946500778198, 0.06554868817329407, 0.0], [0.41325902938842773, 0.09055528044700623, 0.05286572128534317, 0.17467984557151794, 0.17384859919548035, 0.0947914645075798]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9629756212234497, 0.03702434524893761, 0.0, 0.0, 0.0, 0.0], [0.9756426811218262, 0.019658586010336876, 0.00469872634857893, 0.0, 0.0, 0.0], [0.9775736331939697, 0.013286303728818893, 0.002559040440246463, 0.006581075489521027, 0.0, 0.0], [0.9870141744613647, 0.007388271391391754, 0.000957920856308192, 0.001831832341849804, 0.0028077582828700542, 0.0], [0.9409245848655701, 0.016633689403533936, 0.0022979099303483963, 0.005890651606023312, 0.005512924864888191, 0.02874019183218479]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628270864486694, 0.03717285394668579, 0.0, 0.0, 0.0, 0.0], [0.9582237601280212, 0.024641864001750946, 0.017134418711066246, 0.0, 0.0, 0.0], [0.9351300001144409, 0.015331589616835117, 0.014810995198786259, 0.03472748026251793, 0.0, 0.0], [0.9225173592567444, 0.010528748854994774, 0.011010152287781239, 0.019440028816461563, 0.03650377318263054, 0.0], [0.8420167565345764, 0.04357195645570755, 0.007488266099244356, 0.01496153511106968, 0.023852793499827385, 0.06810864061117172]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9926387071609497, 0.007361333351582289, 0.0, 0.0, 0.0, 0.0], [0.9957393407821655, 0.003346970770508051, 0.00091368897119537, 0.0, 0.0, 0.0], [0.9869900345802307, 0.0019747859332710505, 0.0015245546819642186, 0.009510686621069908, 0.0, 0.0], [0.9933527708053589, 0.001020328258164227, 0.00034337403485551476, 0.001029114704579115, 0.004254393745213747, 0.0], [0.9749016761779785, 0.0004348019137978554, 0.000430655520176515, 0.0012364384019747376, 0.001534773618914187, 0.021461637690663338]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9897475242614746, 0.010252486914396286, 0.0, 0.0, 0.0, 0.0], [0.9790639281272888, 0.016509056091308594, 0.00442709494382143, 0.0, 0.0, 0.0], [0.9521436095237732, 0.02943236567080021, 0.008943174965679646, 0.009480932727456093, 0.0, 0.0], [0.9395942091941833, 0.021510910242795944, 0.010278534144163132, 0.004555221181362867, 0.024061065167188644, 0.0], [0.9205074310302734, 0.01615365780889988, 0.010818609036505222, 0.01664443127810955, 0.014566411264240742, 0.02130948379635811]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9898502826690674, 0.01014977227896452, 0.0, 0.0, 0.0, 0.0], [0.9820910096168518, 0.0069075338542461395, 0.011001535691320896, 0.0, 0.0, 0.0], [0.9684997200965881, 0.0089876102283597, 0.015342569909989834, 0.007170100696384907, 0.0, 0.0], [0.9274120330810547, 0.009485269896686077, 0.02206611819565296, 0.03222890570759773, 0.008807665668427944, 0.0], [0.9006660580635071, 0.02162371575832367, 0.013808215968310833, 0.009843839332461357, 0.008521351031959057, 0.04553685709834099]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954444169998169, 0.004555594641715288, 0.0, 0.0, 0.0, 0.0], [0.995254397392273, 0.002460234332829714, 0.0022854835260659456, 0.0, 0.0, 0.0], [0.9862446188926697, 0.0015168144600465894, 0.0040722922421991825, 0.008166342042386532, 0.0, 0.0], [0.9889963865280151, 0.0012260458897799253, 0.0007996382773853838, 0.0006774249486625195, 0.008300590328872204, 0.0], [0.9865202903747559, 0.0003942706680390984, 0.0009571767877787352, 0.0004954360192641616, 0.0009604952065274119, 0.010672261007130146]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9821295142173767, 0.017870519310235977, 0.0, 0.0, 0.0, 0.0], [0.7489445805549622, 0.22002629935741425, 0.03102915547788143, 0.0, 0.0, 0.0], [0.2854783236980438, 0.21125619113445282, 0.47871607542037964, 0.024549338966608047, 0.0, 0.0], [0.8056642413139343, 0.02697453647851944, 0.04302811250090599, 0.06993735581636429, 0.05439591035246849, 0.0], [0.330721914768219, 0.022326510399580002, 0.016627032309770584, 0.080194391310215, 0.41574740409851074, 0.13438260555267334]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9697746634483337, 0.030225370079278946, 0.0, 0.0, 0.0, 0.0], [0.9800565838813782, 0.01501889992505312, 0.004924521781504154, 0.0, 0.0, 0.0], [0.9237861037254333, 0.052764732390642166, 0.006302398629486561, 0.017146700993180275, 0.0, 0.0], [0.9451842904090881, 0.03618054464459419, 0.001989212818443775, 0.0039587244391441345, 0.01268730964511633, 0.0], [0.9633325934410095, 0.018663031980395317, 0.0030418476089835167, 0.007070912979543209, 0.0050094276666641235, 0.0028820668812841177]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.987324595451355, 0.012675448320806026, 0.0, 0.0, 0.0, 0.0], [0.9904569983482361, 0.005541946738958359, 0.004001116845756769, 0.0, 0.0, 0.0], [0.9814971685409546, 0.004653455223888159, 0.00372527539730072, 0.010124039836227894, 0.0, 0.0], [0.9744364619255066, 0.004632262047380209, 0.002379997167736292, 0.006518092937767506, 0.012033062055706978, 0.0], [0.9624499678611755, 0.0033743453677743673, 0.0013198527740314603, 0.0017274925485253334, 0.002944671083241701, 0.028183719143271446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9807673692703247, 0.019232625141739845, 0.0, 0.0, 0.0, 0.0], [0.9664247035980225, 0.0154139194637537, 0.01816137507557869, 0.0, 0.0, 0.0], [0.9632683396339417, 0.004538131412118673, 0.0029253889806568623, 0.029268164187669754, 0.0, 0.0], [0.9562349915504456, 0.0012223643716424704, 0.0005304075893945992, 0.008671474643051624, 0.033340781927108765, 0.0], [0.9657101035118103, 0.0009808274917304516, 0.0016686212038621306, 0.0026348272804170847, 0.005866338964551687, 0.02313929796218872]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639714956283569, 0.036028504371643066, 0.0, 0.0, 0.0, 0.0], [0.9562800526618958, 0.033733122050762177, 0.009986838325858116, 0.0, 0.0, 0.0], [0.853999674320221, 0.08073031902313232, 0.03334450349211693, 0.031925544142723083, 0.0, 0.0], [0.9547489881515503, 0.0096050426363945, 0.00414617033675313, 0.002013325458392501, 0.02948645129799843, 0.0], [0.933113694190979, 0.028699707239866257, 0.005477478262037039, 0.006368071772158146, 0.012613070197403431, 0.013728084973990917]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9392993450164795, 0.060700662434101105, 0.0, 0.0, 0.0, 0.0], [0.9298389554023743, 0.061895497143268585, 0.008265494368970394, 0.0, 0.0, 0.0], [0.8471820950508118, 0.09035064280033112, 0.017636114731431007, 0.04483110085129738, 0.0, 0.0], [0.8857702016830444, 0.03918180242180824, 0.007867713458836079, 0.022765880450606346, 0.0444144681096077, 0.0], [0.8563281893730164, 0.10088981688022614, 0.00653143459931016, 0.008485895581543446, 0.0073684388771653175, 0.020396238192915916]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8353262543678284, 0.16467373073101044, 0.0, 0.0, 0.0, 0.0], [0.6160860061645508, 0.3137648403644562, 0.07014918327331543, 0.0, 0.0, 0.0], [0.34316354990005493, 0.27584975957870483, 0.11966027319431305, 0.261326402425766, 0.0, 0.0], [0.5908167958259583, 0.05029088631272316, 0.04166605323553085, 0.2199493646621704, 0.09727690368890762, 0.0], [0.8481416702270508, 0.0631808340549469, 0.014733636751770973, 0.05526723712682724, 0.009014973416924477, 0.009661651216447353]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9627320766448975, 0.03726791962981224, 0.0, 0.0, 0.0, 0.0], [0.7757521867752075, 0.17996282875537872, 0.044284969568252563, 0.0, 0.0, 0.0], [0.6317057609558105, 0.24380743503570557, 0.10925643891096115, 0.015230290591716766, 0.0, 0.0], [0.9539909958839417, 0.01818232797086239, 0.011601817794144154, 0.01229910645633936, 0.003925777971744537, 0.0], [0.40357086062431335, 0.14237552881240845, 0.056611865758895874, 0.19757317006587982, 0.09299169480800629, 0.10687682777643204]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9802619218826294, 0.01973811723291874, 0.0, 0.0, 0.0, 0.0], [0.9873908162117004, 0.007800452411174774, 0.004808676429092884, 0.0, 0.0, 0.0], [0.9283919334411621, 0.00830123946070671, 0.01330562774091959, 0.05000118911266327, 0.0, 0.0], [0.8981053829193115, 0.015591327100992203, 0.01017757598310709, 0.03998705744743347, 0.03613865002989769, 0.0], [0.975350022315979, 0.0003543295315466821, 0.0005866009742021561, 0.0011877480428665876, 0.0010750859510153532, 0.021446382626891136]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9295330047607422, 0.0704670250415802, 0.0, 0.0, 0.0, 0.0], [0.9361505508422852, 0.04116692394018173, 0.022682547569274902, 0.0, 0.0, 0.0], [0.8486821055412292, 0.05802806839346886, 0.024856165051460266, 0.0684337243437767, 0.0, 0.0], [0.8661178946495056, 0.02232472598552704, 0.010369138792157173, 0.026001976802945137, 0.07518625259399414, 0.0], [0.8074422478675842, 0.04438253492116928, 0.01849709264934063, 0.03357789292931557, 0.0185612291097641, 0.07753899693489075]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9680536389350891, 0.031946372240781784, 0.0, 0.0, 0.0, 0.0], [0.9693689942359924, 0.025684954598546028, 0.004946068394929171, 0.0, 0.0, 0.0], [0.9620568156242371, 0.02255246415734291, 0.005471329670399427, 0.00991946179419756, 0.0, 0.0], [0.9727528095245361, 0.010137127712368965, 0.0007573263137601316, 0.0028828911017626524, 0.013469814322888851, 0.0], [0.9624637365341187, 0.00311090424656868, 0.00100075569935143, 0.0019475899171084166, 0.008266208693385124, 0.023210842162370682]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8542505502700806, 0.14574943482875824, 0.0, 0.0, 0.0, 0.0], [0.9725967645645142, 0.014116342179477215, 0.013286862522363663, 0.0, 0.0, 0.0], [0.9257622957229614, 0.03257261589169502, 0.014612065628170967, 0.02705308608710766, 0.0, 0.0], [0.7923429012298584, 0.027305010706186295, 0.01880667358636856, 0.1385413110256195, 0.02300402708351612, 0.0], [0.6152051091194153, 0.026655299589037895, 0.029352962970733643, 0.0559089221060276, 0.11611286550760269, 0.15676498413085938]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9804654121398926, 0.019534558057785034, 0.0, 0.0, 0.0, 0.0], [0.9882452487945557, 0.0075094737112522125, 0.004245333839207888, 0.0, 0.0, 0.0], [0.9584206342697144, 0.010963579639792442, 0.010456085205078125, 0.020159730687737465, 0.0, 0.0], [0.9604810476303101, 0.007182636763900518, 0.003072344930842519, 0.0068989223800599575, 0.022365106269717216, 0.0], [0.966888964176178, 0.0032812876161187887, 0.005500528495758772, 0.004234079271554947, 0.005038034170866013, 0.015057181939482689]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9498193860054016, 0.050180625170469284, 0.0, 0.0, 0.0, 0.0], [0.9781363606452942, 0.016430046409368515, 0.005433559417724609, 0.0, 0.0, 0.0], [0.8618695139884949, 0.03609364852309227, 0.07555560022592545, 0.026481211185455322, 0.0, 0.0], [0.5449836850166321, 0.015411168336868286, 0.02351653389632702, 0.25743618607521057, 0.1586524099111557, 0.0], [0.9571872353553772, 0.0030803889967501163, 0.001444686553440988, 0.006861576810479164, 0.014818795025348663, 0.01660728268325329]]], [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6156561374664307, 0.3843438923358917, 0.0, 0.0, 0.0, 0.0], [0.36760637164115906, 0.4281637370586395, 0.20422989130020142, 0.0, 0.0, 0.0], [0.16471558809280396, 0.4136791527271271, 0.25092360377311707, 0.1706816852092743, 0.0, 0.0], [0.4184460937976837, 0.15247608721256256, 0.10305383056402206, 0.11071506887674332, 0.21530893445014954, 0.0], [0.19686944782733917, 0.2014620304107666, 0.12827250361442566, 0.0920325294137001, 0.09167566150426865, 0.28968778252601624]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9027365446090698, 0.09726346284151077, 0.0, 0.0, 0.0, 0.0], [0.9736634492874146, 0.014004308730363846, 0.012332303449511528, 0.0, 0.0, 0.0], [0.8504457473754883, 0.05690574273467064, 0.03206087276339531, 0.06058763712644577, 0.0, 0.0], [0.766120970249176, 0.035303935408592224, 0.03433044254779816, 0.09675208479166031, 0.06749249249696732, 0.0], [0.8650376796722412, 0.020085245370864868, 0.0114980423822999, 0.018558325245976448, 0.018430287018418312, 0.06639042496681213]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9653082489967346, 0.03469175100326538, 0.0, 0.0, 0.0, 0.0], [0.9816323518753052, 0.014176092110574245, 0.004191520158201456, 0.0, 0.0, 0.0], [0.9275255799293518, 0.0473722405731678, 0.011528268456459045, 0.013573979958891869, 0.0, 0.0], [0.9293115139007568, 0.025833338499069214, 0.007227125111967325, 0.01430062297731638, 0.023327331990003586, 0.0], [0.8895061612129211, 0.046896226704120636, 0.004717112518846989, 0.006286595948040485, 0.006090151146054268, 0.046503737568855286]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8938026428222656, 0.10619737207889557, 0.0, 0.0, 0.0, 0.0], [0.8221706748008728, 0.063044972717762, 0.11478440463542938, 0.0, 0.0, 0.0], [0.5047378540039062, 0.15375757217407227, 0.22770363092422485, 0.11380091309547424, 0.0, 0.0], [0.40820738673210144, 0.09066358208656311, 0.11696857213973999, 0.24553203582763672, 0.13862840831279755, 0.0], [0.7291041016578674, 0.06638885289430618, 0.023112762719392776, 0.03110307641327381, 0.05714317411184311, 0.09314802289009094]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9247532486915588, 0.07524676620960236, 0.0, 0.0, 0.0, 0.0], [0.8957375288009644, 0.06989562511444092, 0.03436676785349846, 0.0, 0.0, 0.0], [0.7924938201904297, 0.09601156413555145, 0.055091120302677155, 0.056403499096632004, 0.0, 0.0], [0.7891507148742676, 0.07880303263664246, 0.03840154409408569, 0.05396977812051773, 0.039674971252679825, 0.0], [0.780785858631134, 0.07993540912866592, 0.04253171756863594, 0.03234210982918739, 0.017816951498389244, 0.04658801481127739]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9480886459350586, 0.05191135033965111, 0.0, 0.0, 0.0, 0.0], [0.8636949062347412, 0.047562092542648315, 0.08874304592609406, 0.0, 0.0, 0.0], [0.9341371059417725, 0.022224130108952522, 0.022624509409070015, 0.021014362573623657, 0.0, 0.0], [0.9588144421577454, 0.008020910434424877, 0.004490078892558813, 0.00586229981854558, 0.022812405601143837, 0.0], [0.938591718673706, 0.02122773416340351, 0.0048724752850830555, 0.010940182954072952, 0.009524590335786343, 0.01484343409538269]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9763734340667725, 0.023626603186130524, 0.0, 0.0, 0.0, 0.0], [0.9884802103042603, 0.005189403425902128, 0.006330370437353849, 0.0, 0.0, 0.0], [0.9477092027664185, 0.017985165119171143, 0.0101566007360816, 0.024149015545845032, 0.0, 0.0], [0.967192530632019, 0.006552820093929768, 0.003322787582874298, 0.005563332699239254, 0.017368430271744728, 0.0], [0.9584562182426453, 0.007502953987568617, 0.0051363264210522175, 0.008071641437709332, 0.005997130647301674, 0.014835829846560955]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.884070873260498, 0.11592914164066315, 0.0, 0.0, 0.0, 0.0], [0.993125319480896, 0.0050708153285086155, 0.0018038052367046475, 0.0, 0.0, 0.0], [0.9534158110618591, 0.023829087615013123, 0.007748983334749937, 0.015006095170974731, 0.0, 0.0], [0.9151288270950317, 0.010873124934732914, 0.013190999627113342, 0.011050461791455746, 0.049756716936826706, 0.0], [0.8769674301147461, 0.03385210409760475, 0.008486478589475155, 0.009969156235456467, 0.03468579426407814, 0.036039117723703384]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003709519514814019, 0.999629020690918, 0.0, 0.0, 0.0, 0.0], [6.52503949822858e-05, 0.37378302216529846, 0.6261518001556396, 0.0, 0.0, 0.0], [4.606039874488488e-05, 0.21050886809825897, 0.411596417427063, 0.377848744392395, 0.0, 0.0], [4.7531026211800054e-05, 0.11616962403059006, 0.23264268040657043, 0.3985332250595093, 0.2526068687438965, 0.0], [1.2476449455789407e-06, 0.1481970250606537, 0.15813151001930237, 0.30074411630630493, 0.11939028650522232, 0.273535817861557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.971555769443512, 0.02844420075416565, 0.0, 0.0, 0.0, 0.0], [0.9529063701629639, 0.03233078494668007, 0.014762787148356438, 0.0, 0.0, 0.0], [0.9343128204345703, 0.023512953892350197, 0.020498033612966537, 0.021676233038306236, 0.0, 0.0], [0.9529678225517273, 0.00855142343789339, 0.004359327722340822, 0.008064564317464828, 0.026056958362460136, 0.0], [0.9653593897819519, 0.008487634360790253, 0.003499275306239724, 0.002721574855968356, 0.0032828773837536573, 0.016649343073368073]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8630780577659607, 0.13692186772823334, 0.0, 0.0, 0.0, 0.0], [0.7696157097816467, 0.08513346314430237, 0.14525090157985687, 0.0, 0.0, 0.0], [0.7133336067199707, 0.1017090305685997, 0.11931272596120834, 0.06564465165138245, 0.0, 0.0], [0.7186216711997986, 0.05444302782416344, 0.013868178240954876, 0.0780804380774498, 0.13498668372631073, 0.0], [0.7990152835845947, 0.058055855333805084, 0.009447006508708, 0.017770448699593544, 0.021138517186045647, 0.09457287937402725]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9518988728523254, 0.04810114949941635, 0.0, 0.0, 0.0, 0.0], [0.8580654263496399, 0.029445774853229523, 0.11248882114887238, 0.0, 0.0, 0.0], [0.6577739119529724, 0.08513441681861877, 0.12613077461719513, 0.1309608817100525, 0.0, 0.0], [0.8087368607521057, 0.03230159357190132, 0.018418151885271072, 0.06856140494346619, 0.07198190689086914, 0.0], [0.668330192565918, 0.13281351327896118, 0.021880602464079857, 0.02787742204964161, 0.049234092235565186, 0.09986425936222076]]]], \"left_text\": [\"No\", \",\", \" I\", \" am\", \" your\", \" father\"], \"right_text\": [\"No\", \",\", \" I\", \" am\", \" your\", \" father\"]}], \"default_filter\": \"0\", \"display_mode\": \"dark\", \"root_div_id\": \"bertviz-2fa5f9f15ab7437992290934271cfac7\", \"include_layers\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], \"include_heads\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], \"total_heads\": 12} is a template marker that is replaced by actual params.\n",
              "        const config = {};\n",
              "\n",
              "        const MIN_X = 0;\n",
              "        const MIN_Y = 0;\n",
              "        const DIV_WIDTH = 970;\n",
              "        const THUMBNAIL_PADDING = 5;\n",
              "        const DETAIL_WIDTH = 300;\n",
              "        const DETAIL_ATTENTION_WIDTH = 140;\n",
              "        const DETAIL_BOX_WIDTH = 80;\n",
              "        const DETAIL_BOX_HEIGHT = 18;\n",
              "        const DETAIL_PADDING = 15;\n",
              "        const ATTN_PADDING = 0;\n",
              "        const DETAIL_HEADING_HEIGHT = 25;\n",
              "        const HEADING_TEXT_SIZE = 15;\n",
              "        const HEADING_PADDING = 5;\n",
              "        const TEXT_SIZE = 13;\n",
              "        const TEXT_PADDING = 5;\n",
              "        const LAYER_COLORS = d3.schemeCategory10;\n",
              "        const PALETTE = {\n",
              "            'light': {\n",
              "                'text': 'black',\n",
              "                'background': 'white',\n",
              "                'highlight': '#F5F5F5'\n",
              "            },\n",
              "            'dark': {\n",
              "                'text': '#ccc',\n",
              "                'background': 'black',\n",
              "                'highlight': '#222'\n",
              "            }\n",
              "        }\n",
              "\n",
              "        function render() {\n",
              "\n",
              "            // Set global state variables\n",
              "\n",
              "            var attData = config.attention[config.filter];\n",
              "            config.leftText = attData.left_text;\n",
              "            config.rightText = attData.right_text;\n",
              "            config.attn = attData.attn;\n",
              "            config.numLayers = config.attn.length;\n",
              "            config.numHeads = config.attn[0].length;\n",
              "            config.thumbnailBoxHeight = 7 * (12 / config.totalHeads);\n",
              "            const axisSize = HEADING_TEXT_SIZE + HEADING_PADDING + TEXT_SIZE + TEXT_PADDING;\n",
              "            config.thumbnailHeight = Math.max(config.leftText.length, config.rightText.length) * config.thumbnailBoxHeight + 2 * THUMBNAIL_PADDING;\n",
              "            config.thumbnailWidth = (DIV_WIDTH - axisSize) / config.totalHeads;\n",
              "            config.detailHeight = Math.max(config.leftText.length, config.rightText.length) * DETAIL_BOX_HEIGHT + 2 * DETAIL_PADDING + DETAIL_HEADING_HEIGHT;\n",
              "            config.divHeight = Math.max(config.numLayers * config.thumbnailHeight + axisSize, config.detailHeight);\n",
              "\n",
              "            const vis = $(`#${config.rootDivId} #vis`)\n",
              "            vis.empty();\n",
              "            vis.attr(\"height\", config.divHeight);\n",
              "            config.svg = d3.select(`#${config.rootDivId} #vis`)\n",
              "                .append('svg')\n",
              "                .attr(\"width\", DIV_WIDTH)\n",
              "                .attr(\"height\", config.divHeight)\n",
              "                .attr(\"fill\", getBackgroundColor());\n",
              "\n",
              "            renderAxisLabels();\n",
              "\n",
              "            var i;\n",
              "            var j;\n",
              "            for (i = 0; i < config.numLayers; i++) {\n",
              "                for (j = 0; j < config.numHeads; j++) {\n",
              "                    renderThumbnail(i, j);\n",
              "                }\n",
              "            }\n",
              "        }\n",
              "\n",
              "        function renderAxisLabels() {\n",
              "            const axisSize = HEADING_TEXT_SIZE + HEADING_PADDING + TEXT_SIZE + TEXT_PADDING;\n",
              "            const tableWidth = config.thumbnailWidth * config.heads.length;\n",
              "            config.svg.append(\"text\")\n",
              "                .text(\"Heads\")\n",
              "                .attr(\"fill\", \"black\")\n",
              "                .attr(\"font-weight\", \"bold\")\n",
              "                .attr(\"font-size\", HEADING_TEXT_SIZE + \"px\")\n",
              "                .attr(\"x\", axisSize + tableWidth / 2)\n",
              "                .attr(\"text-anchor\", \"middle\")\n",
              "                .attr(\"y\", 0)\n",
              "                .attr(\"dy\", HEADING_TEXT_SIZE);\n",
              "            for (let i = 0; i < config.numHeads; i++) {\n",
              "                config.svg.append(\"text\")\n",
              "                    .text(config.heads[i])\n",
              "                    .attr(\"fill\", \"black\")\n",
              "                    .attr(\"font-size\", TEXT_SIZE + \"px\")\n",
              "                    .attr(\"x\", axisSize + (i + .5) * config.thumbnailWidth)\n",
              "                    .attr(\"text-anchor\", \"middle\")\n",
              "                    .attr(\"y\", HEADING_TEXT_SIZE + HEADING_PADDING)\n",
              "                    .attr(\"dy\", TEXT_SIZE);\n",
              "            }\n",
              "            let x = 0;\n",
              "            let y = axisSize + config.thumbnailHeight * config.layers.length / 2;\n",
              "            console.log(\"x\", x, y)\n",
              "            config.svg.append(\"text\")\n",
              "                .text(\"Layers\")\n",
              "                .attr(\"fill\", \"black\")\n",
              "                .attr(\"font-weight\", \"bold\")\n",
              "                .attr(\"transform\", \"rotate(270, \" + x  + \", \" + y + \")\")\n",
              "                .attr(\"font-size\", HEADING_TEXT_SIZE + \"px\")\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"text-anchor\", \"middle\")\n",
              "                .attr(\"y\", y)\n",
              "                .attr(\"dy\", HEADING_TEXT_SIZE);\n",
              "            for (let i = 0; i < config.numLayers; i++) {\n",
              "                x = HEADING_TEXT_SIZE + HEADING_PADDING + TEXT_SIZE; // HACK\n",
              "                y = axisSize + (i + .5) * config.thumbnailHeight;\n",
              "                config.svg.append(\"text\")\n",
              "                    .text(config.layers[i])\n",
              "                    .attr(\"fill\", \"black\")\n",
              "                    .attr(\"font-size\", TEXT_SIZE + \"px\")\n",
              "                    .attr(\"x\", x)\n",
              "                    .attr(\"text-anchor\", \"end\")\n",
              "                    .attr(\"y\", y)\n",
              "                    .attr(\"dy\", TEXT_SIZE / 2);\n",
              "            }\n",
              "        }\n",
              "\n",
              "\n",
              "        function renderThumbnail(layerIndex, headIndex) {\n",
              "            const axisSize = HEADING_TEXT_SIZE + HEADING_PADDING + TEXT_SIZE + TEXT_PADDING\n",
              "            const x = headIndex * config.thumbnailWidth + axisSize;\n",
              "            const y = layerIndex * config.thumbnailHeight + axisSize;\n",
              "            renderThumbnailAttn(x, y, config.attn[layerIndex][headIndex], layerIndex, headIndex);\n",
              "        }\n",
              "\n",
              "        function renderDetail(att, layerIndex, headIndex) {\n",
              "            const axisSize = TEXT_SIZE + HEADING_PADDING + TEXT_SIZE + TEXT_PADDING;\n",
              "            var xOffset = .8 * config.thumbnailWidth;\n",
              "            var maxX = DIV_WIDTH;\n",
              "            var maxY = config.divHeight - 3;\n",
              "            var leftPos = axisSize + headIndex * config.thumbnailWidth;\n",
              "            var x = leftPos + THUMBNAIL_PADDING + xOffset;\n",
              "            if (x < MIN_X) {\n",
              "                x = MIN_X;\n",
              "            } else if (x + DETAIL_WIDTH > maxX) {\n",
              "                x = leftPos + THUMBNAIL_PADDING - DETAIL_WIDTH + 8;\n",
              "            }\n",
              "            var posLeftText = x;\n",
              "            var posAttention = posLeftText + DETAIL_BOX_WIDTH;\n",
              "            var posRightText = posAttention + DETAIL_ATTENTION_WIDTH;\n",
              "            var thumbnailHeight = Math.max(config.leftText.length, config.rightText.length) * config.thumbnailBoxHeight + 2 * THUMBNAIL_PADDING;\n",
              "            var yOffset = 20;\n",
              "            var y = layerIndex * thumbnailHeight + THUMBNAIL_PADDING + yOffset;\n",
              "            if (y < MIN_Y) {\n",
              "                y = MIN_Y;\n",
              "            } else if (y + config.detailHeight > maxY) {\n",
              "                y = maxY - config.detailHeight;\n",
              "            }\n",
              "            renderDetailFrame(x, y, layerIndex);\n",
              "            y = y + DETAIL_PADDING;\n",
              "            renderDetailHeading(x, y, layerIndex, headIndex);\n",
              "            y = y + DETAIL_HEADING_HEIGHT;\n",
              "            renderDetailText(config.leftText, \"leftText\", posLeftText, y , layerIndex);\n",
              "            renderDetailAttn(posAttention, y, att, layerIndex, headIndex);\n",
              "            renderDetailText(config.rightText, \"rightText\", posRightText, y, layerIndex);\n",
              "        }\n",
              "\n",
              "        function renderDetailHeading(x, y, layerIndex, headIndex) {\n",
              "            var fillColor = getTextColor();\n",
              "            config.svg.append(\"text\")\n",
              "                .classed(\"detail\", true)\n",
              "                .text('Layer ' + config.layers[layerIndex] + \", Head \" + config.heads[headIndex])\n",
              "                .attr(\"font-size\", TEXT_SIZE + \"px\")\n",
              "                .attr(\"font-weight\", \"bold\")\n",
              "                .style(\"cursor\", \"default\")\n",
              "                .style(\"-webkit-user-select\", \"none\")\n",
              "                .attr(\"fill\", fillColor)\n",
              "                .attr(\"x\", x + DETAIL_WIDTH / 2)\n",
              "                .attr(\"text-anchor\", \"middle\")\n",
              "                .attr(\"y\", y)\n",
              "                .attr(\"height\", DETAIL_HEADING_HEIGHT)\n",
              "                .attr(\"width\", DETAIL_WIDTH)\n",
              "                .attr(\"dy\", HEADING_TEXT_SIZE);\n",
              "        }\n",
              "\n",
              "        function renderDetailText(text, id, x, y, layerIndex) {\n",
              "            var tokenContainer = config.svg.append(\"svg:g\")\n",
              "                .classed(\"detail\", true)\n",
              "                .selectAll(\"g\")\n",
              "                .data(text)\n",
              "                .enter()\n",
              "                .append(\"g\");\n",
              "\n",
              "            var fillColor = getTextColor();\n",
              "\n",
              "            tokenContainer.append(\"rect\")\n",
              "                .classed(\"highlight\", true)\n",
              "                .attr(\"fill\", fillColor)\n",
              "                .style(\"opacity\", 0.0)\n",
              "                .attr(\"height\", DETAIL_BOX_HEIGHT)\n",
              "                .attr(\"width\", DETAIL_BOX_WIDTH)\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"y\", function (d, i) {\n",
              "                    return y + i * DETAIL_BOX_HEIGHT;\n",
              "                });\n",
              "\n",
              "            var textContainer = tokenContainer.append(\"text\")\n",
              "                .classed(\"token\", true)\n",
              "                .text(function (d) {\n",
              "                    return d;\n",
              "                })\n",
              "                .attr(\"font-size\", TEXT_SIZE + \"px\")\n",
              "                .style(\"cursor\", \"default\")\n",
              "                .style(\"-webkit-user-select\", \"none\")\n",
              "                .attr(\"fill\", fillColor)\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"y\", function (d, i) {\n",
              "                    return i * DETAIL_BOX_HEIGHT + y;\n",
              "                })\n",
              "                .attr(\"height\", DETAIL_BOX_HEIGHT)\n",
              "                .attr(\"width\", DETAIL_BOX_WIDTH)\n",
              "                .attr(\"dy\", TEXT_SIZE);\n",
              "\n",
              "            if (id == \"leftText\") {\n",
              "                textContainer.style(\"text-anchor\", \"end\")\n",
              "                    .attr(\"dx\", DETAIL_BOX_WIDTH - 2);\n",
              "                tokenContainer.on(\"mouseover\", function (d, index) {\n",
              "                    highlightSelection(index);\n",
              "                });\n",
              "                tokenContainer.on(\"mouseleave\", function () {\n",
              "                    unhighlightSelection();\n",
              "                });\n",
              "            }\n",
              "        }\n",
              "\n",
              "        function highlightSelection(index) {\n",
              "            config.svg.select(\"#leftText\")\n",
              "                .selectAll(\".highlight\")\n",
              "                .style(\"opacity\", function (d, i) {\n",
              "                    return i == index ? 1.0 : 0.0;\n",
              "                });\n",
              "            config.svg.selectAll(\".attn-line-group\")\n",
              "                .style(\"opacity\", function (d, i) {\n",
              "                    return i == index ? 1.0 : 0.0;\n",
              "                });\n",
              "        }\n",
              "\n",
              "        function unhighlightSelection() {\n",
              "            config.svg.select(\"#leftText\")\n",
              "                .selectAll(\".highlight\")\n",
              "                .style(\"opacity\", 0.0);\n",
              "            config.svg.selectAll(\".attn-line-group\")\n",
              "                .style(\"opacity\", 1);\n",
              "        }\n",
              "\n",
              "        function renderThumbnailAttn(x, y, att, layerIndex, headIndex) {\n",
              "\n",
              "            var attnContainer = config.svg.append(\"svg:g\");\n",
              "\n",
              "            var attnBackground = attnContainer.append(\"rect\")\n",
              "                .attr(\"id\", 'attn_background_' + layerIndex + \"_\" + headIndex)\n",
              "                .classed(\"attn_background\", true)\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"y\", y)\n",
              "                .attr(\"height\", config.thumbnailHeight)\n",
              "                .attr(\"width\", config.thumbnailWidth)\n",
              "                .attr(\"stroke-width\", 2)\n",
              "                .attr(\"stroke\", getLayerColor(layerIndex))\n",
              "                .attr(\"stroke-opacity\", 0)\n",
              "                .attr(\"fill\", getBackgroundColor());\n",
              "            var x1 = x + THUMBNAIL_PADDING;\n",
              "            var x2 = x1 + config.thumbnailWidth - 14;\n",
              "            var y1 = y + THUMBNAIL_PADDING;\n",
              "\n",
              "            attnContainer.selectAll(\"g\")\n",
              "                .data(att)\n",
              "                .enter()\n",
              "                .append(\"g\") // Add group for each source token\n",
              "                .attr(\"source-index\", function (d, i) { // Save index of source token\n",
              "                    return i;\n",
              "                })\n",
              "                .selectAll(\"line\")\n",
              "                .data(function (d) { // Loop over all target tokens\n",
              "                    return d;\n",
              "                })\n",
              "                .enter() // When entering\n",
              "                .append(\"line\")\n",
              "                .attr(\"x1\", x1)\n",
              "                .attr(\"y1\", function (d) {\n",
              "                    var sourceIndex = +this.parentNode.getAttribute(\"source-index\");\n",
              "                    return y1 + (sourceIndex + .5) * config.thumbnailBoxHeight;\n",
              "                })\n",
              "                .attr(\"x2\", x2)\n",
              "                .attr(\"y2\", function (d, targetIndex) {\n",
              "                    return y1 + (targetIndex + .5) * config.thumbnailBoxHeight;\n",
              "                })\n",
              "                .attr(\"stroke-width\", 2.2)\n",
              "                .attr(\"stroke\", getLayerColor(layerIndex))\n",
              "                .attr(\"stroke-opacity\", function (d) {\n",
              "                    return d;\n",
              "                });\n",
              "\n",
              "            var clickRegion = attnContainer.append(\"rect\")\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"y\", y)\n",
              "                .attr(\"height\", config.thumbnailHeight)\n",
              "                .attr(\"width\", config.thumbnailWidth)\n",
              "                .style(\"opacity\", 0);\n",
              "\n",
              "            clickRegion.on(\"click\", function (d, index) {\n",
              "                var attnBackgroundOther = config.svg.selectAll(\".attn_background\");\n",
              "                attnBackgroundOther.attr(\"fill\", getBackgroundColor());\n",
              "                attnBackgroundOther.attr(\"stroke-opacity\", 0);\n",
              "\n",
              "                config.svg.selectAll(\".detail\").remove();\n",
              "                if (config.detail_layer != layerIndex || config.detail_head != headIndex) {\n",
              "                    renderDetail(att, layerIndex, headIndex);\n",
              "                    config.detail_layer = layerIndex;\n",
              "                    config.detail_head = headIndex;\n",
              "                    attnBackground.attr(\"fill\", getHighlightColor());\n",
              "                    attnBackground.attr(\"stroke-opacity\", .8);\n",
              "                } else {\n",
              "                    config.detail_layer = null;\n",
              "                    config.detail_head = null;\n",
              "                    attnBackground.attr(\"fill\", getBackgroundColor());\n",
              "                    attnBackground.attr(\"stroke-opacity\", 0);\n",
              "                }\n",
              "            });\n",
              "\n",
              "            clickRegion.on(\"mouseover\", function (d) {\n",
              "                d3.select(this).style(\"cursor\", \"pointer\");\n",
              "            });\n",
              "        }\n",
              "\n",
              "        function renderDetailFrame(x, y, layerIndex) {\n",
              "            var detailFrame = config.svg.append(\"rect\")\n",
              "                .classed(\"detail\", true)\n",
              "                .attr(\"x\", x)\n",
              "                .attr(\"y\", y)\n",
              "                .attr(\"height\", config.detailHeight)\n",
              "                .attr(\"width\", DETAIL_WIDTH)\n",
              "                .style(\"opacity\", 1)\n",
              "                .attr(\"stroke-width\", 1.5)\n",
              "                .attr(\"stroke-opacity\", 0.7)\n",
              "                .attr(\"stroke\", getLayerColor(layerIndex));\n",
              "        }\n",
              "\n",
              "        function renderDetailAttn(x, y, att, layerIndex) {\n",
              "            var attnContainer = config.svg.append(\"svg:g\")\n",
              "                .classed(\"detail\", true)\n",
              "                .attr(\"pointer-events\", \"none\");\n",
              "            attnContainer.selectAll(\"g\")\n",
              "                .data(att)\n",
              "                .enter()\n",
              "                .append(\"g\") // Add group for each source token\n",
              "                .classed('attn-line-group', true)\n",
              "                .attr(\"source-index\", function (d, i) { // Save index of source token\n",
              "                    return i;\n",
              "                })\n",
              "                .selectAll(\"line\")\n",
              "                .data(function (d) { // Loop over all target tokens\n",
              "                    return d;\n",
              "                })\n",
              "                .enter()\n",
              "                .append(\"line\")\n",
              "                .attr(\"x1\", x + ATTN_PADDING)\n",
              "                .attr(\"y1\", function (d) {\n",
              "                    var sourceIndex = +this.parentNode.getAttribute(\"source-index\");\n",
              "                    return y + (sourceIndex + .5) * DETAIL_BOX_HEIGHT;\n",
              "                })\n",
              "                .attr(\"x2\", x + DETAIL_ATTENTION_WIDTH - ATTN_PADDING)\n",
              "                .attr(\"y2\", function (d, targetIndex) {\n",
              "                    return y + (targetIndex + .5) * DETAIL_BOX_HEIGHT;\n",
              "                })\n",
              "                .attr(\"stroke-width\", 2.2)\n",
              "                .attr(\"stroke\", getLayerColor(layerIndex))\n",
              "                .attr(\"stroke-opacity\", function (d) {\n",
              "                    return d;\n",
              "                });\n",
              "        }\n",
              "\n",
              "        function getLayerColor(layer) {\n",
              "          return LAYER_COLORS[config.layers[layer] % 10];\n",
              "        }\n",
              "\n",
              "        function getTextColor() {\n",
              "            return PALETTE[config.mode]['text']\n",
              "        }\n",
              "\n",
              "        function getBackgroundColor() {\n",
              "           return PALETTE[config.mode]['background']\n",
              "        }\n",
              "\n",
              "        function getHighlightColor() {\n",
              "           return PALETTE[config.mode]['highlight']\n",
              "        }\n",
              "\n",
              "        function initialize() {\n",
              "            config.attention = params['attention'];\n",
              "            config.filter = params['default_filter'];\n",
              "            config.mode = params['display_mode'];\n",
              "            config.layers = params['include_layers']\n",
              "            config.heads = params['include_heads']\n",
              "            config.totalHeads = params['total_heads']\n",
              "            config.rootDivId = params['root_div_id'];\n",
              "            $(`#${config.rootDivId} #filter`).on('change', function (e) {\n",
              "                config.filter = e.currentTarget.value;\n",
              "                render();\n",
              "            });\n",
              "        }\n",
              "\n",
              "        initialize();\n",
              "        render();\n",
              "\n",
              "    });"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pipeline using HuggingFace\n",
        "\n",
        "Now, let's see a practical application of LLMs using a HuggingFace pipeline for classification.\n",
        "\n",
        "This involves a few steps including:\n",
        "1. Setting up a prompt\n",
        "2. Loading in a pretrained model\n",
        "3. Loading in the tokenizer and tokenizing input text\n",
        "4. Performing model inference\n",
        "5. Interpreting inference output"
      ],
      "metadata": {
        "id": "u1FxXk7_zbOH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# STEP 0 : Installations and imports\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig\n",
        "import torch\n",
        "import torch.nn.functional as F"
      ],
      "metadata": {
        "id": "uBQbE6nv0fpO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Setting up a prompt"
      ],
      "metadata": {
        "id": "4jRW20eG0PAn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "A \"prompt\" refers to a specific input or query provided to a language model. They guide the text processing and generation by providing the context for the model to generate coherent and relevant text based on the given input.\n",
        "\n",
        "The choice and structure of the prompt depends on the specific task, the context and desired output. Prompts can be \"discrete\" or \"instructive\" where they are explicit instructions or questions directed to the language model. They can also be more nuanced by more providing suggestions, directions and contexts to the model.\n",
        "\n",
        "We will use very simple prompts in this tutorial section, but we will learn more about prompt engineering and how it helps in optimizing the performance of the model for a given use case in the following tutorials."
      ],
      "metadata": {
        "id": "afMWePMW0mba"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# STEP 1 : Set up the prompt\n",
        "input_text = \"The panoramic view of the ocean was breathtaking.\""
      ],
      "metadata": {
        "id": "y9b6ARMj0NeE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 2. Loading Pretrained Models\n",
        "\n",
        "The AutoModelForSequenceClassification from_pretrained() method instantiates a sequence classification model.\n",
        "\n",
        "Refer to https://huggingface.co/transformers/v3.0.2/model_doc/auto.html#automodels for the list of model classes supported.\n",
        "\n",
        "\"from_pretrained\" method downloads the pre-trained weights from the Hugging Face Model Hub or the specified URL if the model is not already cached locally. It then loads the weights into the instantiated model, initializing the model parameters with the pre-trained values.\n",
        "\n",
        "The model cache contains:\n",
        "\n",
        "* model configuration (config.json)\n",
        "* pretrained model weights (model.safetensors)\n",
        "* tokenizer information (tokenizer.json, vocab.json, merges.txt, tokenizer.model)"
      ],
      "metadata": {
        "id": "C4KEMFkf1mEl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# STEP 2 : Load the pretrained model.\n",
        "model_name = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
        "config = AutoConfig.from_pretrained(model_name)\n",
        "print(config)"
      ],
      "metadata": {
        "id": "A8feWbI41sCd",
        "outputId": "ffbbbf94-7b6e-4641-f11c-eca4fcd149c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 671,
          "referenced_widgets": [
            "bd858574deec4181bb0ba443be9a2088",
            "b025f1fa9b61436f93515742e324b86f",
            "5073ff96876f4e04b246e466be18c186",
            "0c9e9d1ae5ed4ae4a7e37513d35f9c77",
            "b29ed3c3e599460cbd3d693d71f456d2",
            "5b43c22a0fa94a7690d637dcef8f9e0a",
            "51736c335dd349e7bfcc529ada3c6f6a",
            "4791ebe7b3cc4444afdcbfd8d1b5967a",
            "93c49acd409c4616b22b28ef971c9dd6",
            "4777bd67c1e64692b294830f05c7143d",
            "aeb6c3883112462c8ade793f34ebc4da",
            "82ebac62b0954db1b9fa3e3ae4025a04",
            "88bc6af07dc244b8bb3ea694ed31d2ae",
            "21364965ba974886ad36c5d9670e7a70",
            "00c33e96a50b4993a9f3387402fbccfe",
            "9de7b337c654426fac8f86b97c7ca7dd",
            "d3d454f41f164fc5af66adf8d2caa53c",
            "06cd5703c7134fd49a66a259d0154b85",
            "f6131609549c4e7eb89cbb3f430d8aca",
            "e5a7799affd54336b99042ae28609630",
            "5ee4ce555b264be4becae07722f88a25",
            "4e181643cf4e449ca06776ecbba79c5e"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd858574deec4181bb0ba443be9a2088"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "82ebac62b0954db1b9fa3e3ae4025a04"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DistilBertConfig {\n",
            "  \"_name_or_path\": \"distilbert-base-uncased-finetuned-sst-2-english\",\n",
            "  \"activation\": \"gelu\",\n",
            "  \"architectures\": [\n",
            "    \"DistilBertForSequenceClassification\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"finetuning_task\": \"sst-2\",\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"NEGATIVE\",\n",
            "    \"1\": \"POSITIVE\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"label2id\": {\n",
            "    \"NEGATIVE\": 0,\n",
            "    \"POSITIVE\": 1\n",
            "  },\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.44.2\",\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 3. Loading in the tokenizer and tokenizing input text\n",
        "\n",
        "Here, we load in a pretrained tokenizer associated with this model."
      ],
      "metadata": {
        "id": "oPFIax-A2FZ2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#STEP 3 : Load the tokenizer and tokenize the input text\n",
        "tokenizer  =  AutoTokenizer.from_pretrained(model_name)\n",
        "input_ids = tokenizer(input_text, return_tensors=\"pt\")[\"input_ids\"]\n",
        "print(input_ids)"
      ],
      "metadata": {
        "id": "cVyVrzoC2XlE",
        "outputId": "431ee7a6-7081-4eb1-a71b-d5db981d1d2b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "067f49dd461a44fabf2b2fbc48c38ea3",
            "8081fe15a05f4d93a263016d65f7c867",
            "db4f4633a272445fb65b31b81cc13a47",
            "683187173eee4b4f84c42ee1b030041e",
            "9b356d4bb2584404b40a53e84ffd08ad",
            "f8844d328728454cabd0c601f1dfac4c",
            "999973bfe8554a6b89b2214b39348331",
            "80f90f7e6d644d1d8b7df13c9e85225d",
            "683ed139b6384e78a7d16d8375640c74",
            "aaca325b53f449519e4193981a4b3865",
            "d1a67e3364a04697a740fac83669d9f9",
            "9489672461b54621bdb8bb24d0eb9934",
            "2862c54dff8c4f83829e946c69962221",
            "95367cef20634367b463b8f0a95e0503",
            "fac8d31cc40e4c9e80f8ca93a9075077",
            "e61474e5f4784cf6ab8a47decc035ae0",
            "b28301ee0a554b009b4a04ab5c3bcaf0",
            "8a29da2807084308bd8ccf9414150e31",
            "545d6ab11dc744099b608face7404d0e",
            "6c4a6d9277f148c6a575b1b7c6c008f7",
            "cac928cad64b40cb8d0ef32f3731451a",
            "a0e7b739e9a44660b0b8b65448d89a93"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "067f49dd461a44fabf2b2fbc48c38ea3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9489672461b54621bdb8bb24d0eb9934"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[  101,  1996,  6090,  6525,  7712,  3193,  1997,  1996,  4153,  2001,\n",
            "          3052, 17904,  1012,   102]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4. Performing inference and interpreting\n",
        "\n",
        "Here, we:\n",
        "* load data into the model,\n",
        "* perform inference to obtain logits,\n",
        "* Convert logits into probabilities\n",
        "* According to probabilities assign label\n",
        "\n",
        "The end result is that we can predict whether the input phrase is positive or negative."
      ],
      "metadata": {
        "id": "q3McDP8i2t5_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# STEP 5 : Perform inference\n",
        "outputs = model(input_ids)\n",
        "result = outputs.logits\n",
        "print(result)\n",
        "\n",
        "# STEP 6 :  Interpret the output.\n",
        "probabilities = F.softmax(result, dim=-1)\n",
        "print(probabilities)\n",
        "predicted_class = torch.argmax(probabilities, dim=-1).item()\n",
        "labels = [\"NEGATIVE\", \"POSITIVE\"]\n",
        "out_string = \"[{'label': '\" + str(labels[predicted_class]) + \"', 'score': \" + str(probabilities[0][predicted_class].tolist()) + \"}]\"\n",
        "print(out_string)"
      ],
      "metadata": {
        "id": "nFuCHG3W2sAU",
        "outputId": "549e2599-545c-49ce-d36f-bec11067c588",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-4.2767,  4.5486]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[1.4695e-04, 9.9985e-01]], grad_fn=<SoftmaxBackward0>)\n",
            "[{'label': 'POSITIVE', 'score': 0.9998530149459839}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Saving and loading models\n",
        "\n",
        "Model can be saved and loaded to and from a local model directory."
      ],
      "metadata": {
        "id": "BNkhKMkT2p3r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModel, AutoModelForCausalLM\n",
        "\n",
        "# Instantiate and train or fine-tune a model\n",
        "model = AutoModelForCausalLM.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "# Train or fine-tune the model...\n",
        "\n",
        "# Save the model to a local directory\n",
        "directory = \"my_local_model\"\n",
        "model.save_pretrained(directory)\n",
        "\n",
        "# Load a pre-trained model from a local directory\n",
        "loaded_model = AutoModel.from_pretrained(directory)"
      ],
      "metadata": {
        "id": "o81LQzDE4kwg",
        "outputId": "80936ea2-2095-4504-f320-03c3f025ff67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "5c584862c1714236be9a4d72eacb7e8c",
            "878ac7474c814c56a1f8e4ca1307b446",
            "61d31a811e704e0b940337009892c774",
            "90d95ae3318b4f4bbda0f5d016347e91",
            "c603255a544f47b280fd61e9e45d9204",
            "bc69805bf372491d931e20940cf99cfc",
            "9eb1cc51fc0348069875ce5c52c992b9",
            "a36bbfb2408944c39232013f53811f08",
            "16994401181f43238336a07a06be119b",
            "dc6892bac378490abd7f628bca35906c",
            "1dcd4ce0732240aab355557623a19d59",
            "3933d404330d4d02913a7e075819f807",
            "f286b77c68ce47ec9a5177f2dd35ea48",
            "d15d08eb7374438f8edbcc2d78beafc8",
            "a9a334e404fe47da819a7e97d0dbf958",
            "ed261fb3eb6b4661a0dba67a3fc9858b",
            "95d2b19696f44d099e530f82ad213328",
            "725a53042a724dc0a9a658bd3dd3cb83",
            "e4dd55f635a74dbeb0067d1c47769123",
            "832adb3112224a51ba9376a50076dd8c",
            "c953196990624b0ea79c2a3c74cd1de9",
            "46f89e0a5ae84c6c97265c795fae9f88"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5c584862c1714236be9a4d72eacb7e8c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3933d404330d4d02913a7e075819f807"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model Hub\n",
        "The Model Hub is where the members of the Hugging Face community can host all of their model checkpoints for simple storage, discovery, and sharing.\n",
        "\n",
        "* Download pre-trained models with the huggingface_hub client library, with Transformers for fine-tuning.\n",
        "* Make use of Inference API to use models in production settings.\n",
        "* You can filter for different models for different tasks, frameworks used, datasets used, and many more.\n",
        "* You can select any model, that will show the model card.\n",
        "* Model card contains information of the model, including the description, usage, limitations etc. Some models also have inference API's that can be used directly.\n",
        "\n",
        "Model Hub Link : https://huggingface.co/docs/hub/en/models-the-hub\n",
        "\n",
        "Example of a model card : https://huggingface.co/bert-base-uncased/tree/main"
      ],
      "metadata": {
        "id": "iVQRB38M44le"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Recommended reading\n",
        "\n",
        "* [\"The Illustrated Transformer\" by Jay Alammar](https://jalammar.github.io/illustrated-transformer/)\n",
        "* [\"Visualizing A Neural Machine Translation Model (Mechanics of Seq2seq Models With Attention)\" by Jay Alammar](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/)\n",
        "* [\"The Illustrated GPT-2 (Visualizing Transformer Language Models)\"](https://jalammar.github.io/illustrated-gpt2/)\n",
        "* [\"A gentle introduction to positional encoding\"](https://machinelearningmastery.com/a-gentle-introduction-to-positional-encoding-in-transformer-models-part-1/)\n",
        "* [\"LLM Tutorial Workshop (Argonne National Laboratory)\"](https://github.com/brettin/llm_tutorial)\n",
        "* [\"LLM Tutorial Workshop Part 2 (Argonne National Laboratory)\"](https://github.com/argonne-lcf/llm-workshop)"
      ],
      "metadata": {
        "id": "5oU3awVy5xzA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Homework\n",
        "\n",
        "1. Load in a generative model using the HuggingFace pipeline and generate text using a batch of prompts.\n",
        "  * Play with generative parameters such as temperature, max_new_tokens, and the model itself and explain the effect on the legibility of the model response. Try at least 4 different parameter/model combinations.\n",
        "  * Models that can be used include:\n",
        "    * `google/gemma-2-2b-it`\n",
        "    * `microsoft/Phi-3-mini-4k-instruct`\n",
        "    * `meta-llama/Llama-3.2-1B`\n",
        "    * Any model from this list: [Text-generation models](https://huggingface.co/models?pipeline_tag=text-generation)\n",
        "    * `gpt2` if having trouble loading these models in\n",
        "  * This guide should help! [Text-generation strategies](https://huggingface.co/docs/transformers/en/generation_strategies)\n",
        "2. Load in 2 models of different parameter size (e.g. GPT2, meta-llama/Llama-2-7b-chat-hf, or distilbert/distilgpt2) and analyze the BertViz for each. How does the attention mechanisms change depending on model size?"
      ],
      "metadata": {
        "id": "oxP6KPNm2ZXz"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ishneA15OLNS"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}