diff --git a/jointContribution/AI_Climate_Diseases/ERA5_land(1).ipynb b/jointContribution/AI_Climate_Diseases/ERA5_land(1).ipynb new file mode 100644 index 0000000000..f2e4bd63f9 --- /dev/null +++ b/jointContribution/AI_Climate_Diseases/ERA5_land(1).ipynb @@ -0,0 +1,1608 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "77f152bb34f64833ad4f9e10337992f6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_79cf5f4c819246149f0b3b7cfaf03b5b", + "IPY_MODEL_c142db7f1d6c4222b3811342b46b0c30", + "IPY_MODEL_485aa27dcfbb4cb9a61da98a8ad277c1" + ], + "layout": "IPY_MODEL_652a7f72be404e2580fb4592348330cb" + } + }, + "79cf5f4c819246149f0b3b7cfaf03b5b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76fd9553926040f5a108afe18be359b8", + "placeholder": "​", + "style": "IPY_MODEL_492b0421e1cc4ee2b02d24910fbbd367", + "value": "e5dc627d8a097bec79906e75846db42e.zip:  90%" + } + }, + "c142db7f1d6c4222b3811342b46b0c30": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4065f75702ea4b30b25b3e0adc6d1cf4", + "max": 4672516, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8d4b92424bde4065b92126463c3a88cf", + "value": 4672516 + } + }, + "485aa27dcfbb4cb9a61da98a8ad277c1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_da3c858f56284927907e863b3f7bff15", + "placeholder": "​", + "style": "IPY_MODEL_17a5e81074e042208e7c41a6fe72d705", + "value": " 4.00M/4.46M [00:01<00:00, 4.23MB/s]" + } + }, + "652a7f72be404e2580fb4592348330cb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": "hidden", + "width": null + } + }, + "76fd9553926040f5a108afe18be359b8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "492b0421e1cc4ee2b02d24910fbbd367": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4065f75702ea4b30b25b3e0adc6d1cf4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8d4b92424bde4065b92126463c3a88cf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "da3c858f56284927907e863b3f7bff15": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "17a5e81074e042208e7c41a6fe72d705": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7638bda1ed3e4c66a4a47c3ff6a8edea": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b30de5c069a4404ab2f48af8feef79cc", + "IPY_MODEL_23894ce319a141d19310e4c02124bd38", + "IPY_MODEL_2bf99ea9822d41d39a8fbd324bb8b3e5" + ], + "layout": "IPY_MODEL_3a5456e2d86a48be853f4552be673819" + } + }, + "b30de5c069a4404ab2f48af8feef79cc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2f8e3c66060b4ef1b57aa2ea97259b7b", + "placeholder": "​", + "style": "IPY_MODEL_daf5ce1cd8e44d47aa090b2693b83d10", + "value": "76d9dacc7921e0bd06d9383480d75e62.zip:  89%" + } + }, + "23894ce319a141d19310e4c02124bd38": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_701ad51f4d414a3fbaa9c9eb5e448785", + "max": 4688001, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5b81063a9d2f46d69a3d138704717817", + "value": 4688001 + } + }, + "2bf99ea9822d41d39a8fbd324bb8b3e5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_87ca252dfca34361ab50ea1809856703", + "placeholder": "​", + "style": "IPY_MODEL_db01f8756847498482428b5460105a6b", + "value": " 4.00M/4.47M [00:01<00:00, 4.38MB/s]" + } + }, + "3a5456e2d86a48be853f4552be673819": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": "hidden", + "width": null + } + }, + "2f8e3c66060b4ef1b57aa2ea97259b7b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "daf5ce1cd8e44d47aa090b2693b83d10": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "701ad51f4d414a3fbaa9c9eb5e448785": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5b81063a9d2f46d69a3d138704717817": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "87ca252dfca34361ab50ea1809856703": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "db01f8756847498482428b5460105a6b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + } + }, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3tsuE-BfSlWm", + "outputId": "d1bd225c-8951-4b46-aa4d-29288258c515" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting cdsapi\n", + " Downloading cdsapi-0.7.6-py2.py3-none-any.whl.metadata (3.0 kB)\n", + "Collecting ecmwf-datastores-client (from cdsapi)\n", + " Downloading ecmwf_datastores_client-0.4.0-py3-none-any.whl.metadata (21 kB)\n", + "Requirement already satisfied: requests>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from cdsapi) (2.32.4)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from cdsapi) (4.67.1)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests>=2.5.0->cdsapi) (3.4.3)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests>=2.5.0->cdsapi) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests>=2.5.0->cdsapi) (2.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests>=2.5.0->cdsapi) (2025.8.3)\n", + "Requirement already satisfied: attrs in /usr/local/lib/python3.12/dist-packages (from ecmwf-datastores-client->cdsapi) (25.3.0)\n", + "Collecting multiurl>=0.3.7 (from ecmwf-datastores-client->cdsapi)\n", + " Downloading multiurl-0.3.7-py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.12/dist-packages (from ecmwf-datastores-client->cdsapi) (4.15.0)\n", + "Requirement already satisfied: pytz in /usr/local/lib/python3.12/dist-packages (from multiurl>=0.3.7->ecmwf-datastores-client->cdsapi) (2025.2)\n", + "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.12/dist-packages (from multiurl>=0.3.7->ecmwf-datastores-client->cdsapi) (2.9.0.post0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil->multiurl>=0.3.7->ecmwf-datastores-client->cdsapi) (1.17.0)\n", + "Downloading cdsapi-0.7.6-py2.py3-none-any.whl (12 kB)\n", + "Downloading ecmwf_datastores_client-0.4.0-py3-none-any.whl (29 kB)\n", + "Downloading multiurl-0.3.7-py3-none-any.whl (21 kB)\n", + "Installing collected packages: multiurl, ecmwf-datastores-client, cdsapi\n", + "Successfully installed cdsapi-0.7.6 ecmwf-datastores-client-0.4.0 multiurl-0.3.7\n" + ] + } + ], + "source": [ + "!pip install cdsapi" + ] + }, + { + "cell_type": "code", + "source": [ + "\n", + "import os, getpass, textwrap, pathlib\n", + "\n", + "\n", + "cfg = textwrap.dedent(f\"\"\"\\\n", + "url: https://cds.climate.copernicus.eu/api\n", + "key: 55a51e6d-554d-46e6-8743-c8f5f4a98f9b\n", + "\"\"\")\n", + "\n", + "path = pathlib.Path(\"~/.cdsapirc\").expanduser()\n", + "path.write_text(cfg)\n", + "# 收紧权限（Linux 600）\n", + "!chmod 600 ~/.cdsapirc\n", + "\n", + "print(\"~/.cdsapirc 写入完成\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mGHbxM7JWnn-", + "outputId": "d7bb263e-bd87-44e6-e29c-c879b263b8ce" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "~/.cdsapirc 写入完成\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import cdsapi\n", + "c = cdsapi.Client()\n", + "print(\"CDS client OK\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "gpHQ393HXAC2", + "outputId": "9468f689-9e15-4a20-e75c-d8ac19012343" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:16:07,307 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:07,309 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "CDS client OK\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# download_era5land_chunked_full_parallel_by_var.py —— 在你的基础上：按“变量”并发\n", + "# -*- coding: utf-8 -*-\n", + "import argparse\n", + "import time\n", + "import random\n", + "from pathlib import Path\n", + "from typing import List, Tuple\n", + "import sys\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "\n", + "import cdsapi\n", + "\n", + "DATASET = \"reanalysis-era5-land\"\n", + "\n", + "# ---------- 时间维度 ----------\n", + "ALL_DAYS: List[str] = [f\"{d:02d}\" for d in range(1, 32)]\n", + "ALL_HOURS: List[str] = [f\"{h:02d}:00\" for h in range(0, 24)]\n", + "ALL_MONTHS: List[str] = [f\"{m:02d}\" for m in range(1, 13)]\n", + "\n", + "# ---------- 重试设置 ----------\n", + "MAX_RETRIES = 8\n", + "BASE_SLEEP = 10 # seconds\n", + "\n", + "# ---------- 变量全集（你的清单 + 注释项全部纳入） ----------\n", + "VARIABLES: List[str] = [\n", + " # 2m/skin/soil/lake temps\n", + " \"2m_dewpoint_temperature\",\n", + " \"2m_temperature\",\n", + " \"skin_temperature\",\n", + " \"soil_temperature_level_1\",\n", + " \"soil_temperature_level_2\",\n", + " \"soil_temperature_level_3\",\n", + " \"soil_temperature_level_4\",\n", + " \"lake_bottom_temperature\",\n", + " \"lake_ice_depth\",\n", + " \"lake_ice_temperature\",\n", + " \"lake_mix_layer_depth\",\n", + " \"lake_mix_layer_temperature\",\n", + " \"lake_shape_factor\",\n", + " # 你原注释掉的项（已启用）\n", + " \"lake_total_layer_temperature\",\n", + " \"snow_albedo\",\n", + " \"snow_cover\",\n", + " \"snow_density\",\n", + " \"snow_depth\",\n", + " \"snow_depth_water_equivalent\",\n", + " \"snowfall\",\n", + " \"snowmelt\",\n", + " \"temperature_of_snow_layer\",\n", + " \"forecast_albedo\",\n", + " \"surface_latent_heat_flux\",\n", + " \"surface_net_solar_radiation\",\n", + " \"surface_net_thermal_radiation\",\n", + " \"surface_sensible_heat_flux\",\n", + " \"surface_solar_radiation_downwards\",\n", + " \"surface_thermal_radiation_downwards\",\n", + " \"evaporation_from_bare_soil\",\n", + " \"evaporation_from_open_water_surfaces_excluding_oceans\",\n", + " \"evaporation_from_the_top_of_canopy\",\n", + " \"evaporation_from_vegetation_transpiration\",\n", + " \"potential_evaporation\",\n", + " \"runoff\",\n", + " \"snow_evaporation\",\n", + " \"sub_surface_runoff\",\n", + " \"surface_runoff\",\n", + " \"total_evaporation\",\n", + " \"10m_u_component_of_wind\",\n", + " \"10m_v_component_of_wind\",\n", + " \"surface_pressure\",\n", + " \"total_precipitation\",\n", + " \"leaf_area_index_high_vegetation\",\n", + " \"leaf_area_index_low_vegetation\",\n", + " \"high_vegetation_cover\",\n", + " \"glacier_mask\",\n", + " \"lake_cover\",\n", + " \"low_vegetation_cover\",\n", + " \"lake_total_depth\",\n", + " \"land_sea_mask\",\n", + " \"soil_type\",\n", + " \"type_of_high_vegetation\",\n", + " \"type_of_low_vegetation\",\n", + "]\n", + "\n", + "def build_request(\n", + " variable: str,\n", + " year: str,\n", + " month: str,\n", + " area_box: Tuple[float, float, float, float],\n", + " fmt: str,\n", + ") -> dict:\n", + " north, west, south, east = area_box\n", + " req = {\n", + " \"variable\": variable,\n", + " \"year\": year,\n", + " \"month\": month,\n", + " \"day\": ALL_DAYS,\n", + " \"time\": ALL_HOURS,\n", + " \"area\": [north, west, south, east], # N W S E\n", + " \"format\": fmt, # \"grib\" | \"netcdf\"\n", + " \"download_format\": \"zip\",\n", + " # \"product_type\": \"reanalysis\",\n", + " }\n", + " return req\n", + "\n", + "def safe_retrieve(client: cdsapi.Client, dataset: str, request: dict, target_path: Path):\n", + " \"\"\"下载单个分块（含指数退避 + 轻度抖动），返回 True/False\"\"\"\n", + " # 轻度抖动，降低“羊群效应”\n", + " time.sleep(random.uniform(0.3, 1.0))\n", + " attempt = 0\n", + " while True:\n", + " try:\n", + " client.retrieve(dataset, request).download(str(target_path))\n", + " return True\n", + " except Exception as e:\n", + " attempt += 1\n", + " msg = str(e).lower()\n", + " # 明确不可恢复的错误（变量无效/不可用/无数据）直接跳过\n", + " unrecoverable_signals = [\n", + " \"unavailable\",\n", + " \"not available\",\n", + " \"invalid\",\n", + " \"does not match\",\n", + " \"no data\",\n", + " \"bad request\",\n", + " \"cannot be found\",\n", + " ]\n", + " if any(s in msg for s in unrecoverable_signals):\n", + " print(f\"[ERROR] Unrecoverable for {target_path.name}: {e}\")\n", + " return False\n", + "\n", + " if attempt > MAX_RETRIES:\n", + " print(f\"[ERROR] Max retries exceeded for {target_path.name}: {e}\")\n", + " return False\n", + "\n", + " sleep_s = BASE_SLEEP * (2 ** (attempt - 1)) * random.uniform(0.85, 1.15)\n", + " print(f\"[WARN] Download failed (attempt {attempt}/{MAX_RETRIES}): {e}\")\n", + " print(f\" Sleeping {sleep_s:.0f}s then retrying...\")\n", + " time.sleep(sleep_s)\n", + "\n", + "def parse_args_with_defaults():\n", + " parser = argparse.ArgumentParser(\n", + " description=\"ERA5-Land downloader (split by variable × year × month), parallel by VARIABLE\"\n", + " )\n", + " # —— 给出默认值，不再强制要求 —— #\n", + " parser.add_argument(\"--out_dir\", type=str, default=\"./era5land\",\n", + " help=\"输出根目录（默认 ./era5land）\")\n", + " parser.add_argument(\"--bbox\", nargs=4, type=float,\n", + " default=[60.86, -6.23, 49.86, 1.75],\n", + " metavar=(\"NORTH\", \"WEST\", \"SOUTH\", \"EAST\"),\n", + " help=\"经纬度范围：N W S E（默认 60.86 -6.23 49.86 1.75）\")\n", + " parser.add_argument(\"--format\", default=\"grib\", choices=[\"grib\", \"netcdf\"],\n", + " help=\"文件格式（默认 grib）\")\n", + " parser.add_argument(\"--years\", nargs=\"+\",\n", + " default=[str(y) for y in range(1997, 2023)], # 1997–2022\n", + " help=\"年份列表（默认 1997..2022）\")\n", + " parser.add_argument(\"--months\", nargs=\"+\", default=ALL_MONTHS,\n", + " help=\"月份列表（默认 01..12）\")\n", + " parser.add_argument(\"--variables\", nargs=\"+\", default=VARIABLES,\n", + " help=\"变量名列表（默认为脚本内置全集）\")\n", + " parser.add_argument(\"--skip_existing\", action=\"store_true\",\n", + " help=\"若目标文件已存在则跳过\")\n", + " parser.add_argument(\"--max_workers\", type=int, default=3,\n", + " help=\"并发的变量数（建议 2–3）\")\n", + " # 如果在 Notebook 中直接运行，且没有传任何参数，也能用默认值\n", + " try:\n", + " return parser.parse_args([])\n", + " except SystemExit:\n", + " # 在某些环境 parse_args([]) 会触发 SystemExit，退回到标准方式\n", + " return parser.parse_args()\n", + "\n", + "def download_one_variable(var: str, args) -> tuple[str, int, int]:\n", + " \"\"\"在一个线程内：顺序下载某个变量的所有 年×月，返回 (var, ok, fail)\"\"\"\n", + " client = cdsapi.Client() # 每个线程各自的 client\n", + " ok = 0\n", + " fail = 0\n", + "\n", + " out_root = Path(args.out_dir)\n", + "\n", + " for year in args.years:\n", + " for month in args.months:\n", + " subdir = out_root / var / str(year)\n", + " subdir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " suffix = \"grib\" if args.format == \"grib\" else \"nc\"\n", + " target_name = f\"{DATASET}_{var}_{year}-{month}.{suffix}.zip\"\n", + " target_path = subdir / target_name\n", + "\n", + " if args.skip_existing and target_path.exists():\n", + " # 已存在直接视为成功，便于断点续跑\n", + " # 你也可以换成校验 zip 完整性的逻辑\n", + " continue_ok = True\n", + " if continue_ok:\n", + " ok += 1\n", + " continue\n", + "\n", + " req = build_request(\n", + " variable=var,\n", + " year=str(year),\n", + " month=f\"{int(month):02d}\",\n", + " area_box=tuple(args.bbox),\n", + " fmt=args.format,\n", + " )\n", + "\n", + " print(f\"[INFO][{var}] Downloading {year}-{month} -> {target_path}\")\n", + " success = safe_retrieve(client, DATASET, req, target_path)\n", + " if success:\n", + " ok += 1\n", + " else:\n", + " fail += 1\n", + "\n", + " return var, ok, fail\n", + "\n", + "def main():\n", + " # 在 Notebook/Colab 里，这里会采用默认值；命令行下可用参数覆盖\n", + " if \"ipykernel\" in sys.modules or \"google.colab\" in sys.modules:\n", + " args = parse_args_with_defaults()\n", + " else:\n", + " args = parse_args_with_defaults()\n", + "\n", + " variables = list(args.variables)\n", + " if not variables:\n", + " print(\"[WARN] 未提供变量列表，使用内置 VARIABLES。\")\n", + " variables = VARIABLES\n", + "\n", + " # 并发数量不超过变量数\n", + " max_workers = max(1, min(args.max_workers, len(variables)))\n", + "\n", + " print(f\"[INFO] Variables: {len(variables)} | Years: {len(args.years)} | Months: {len(args.months)}\")\n", + " print(f\"[INFO] Parallel by VARIABLE with max_workers = {max_workers}\")\n", + " start = time.time()\n", + "\n", + " total_ok = 0\n", + " total_fail = 0\n", + " results = []\n", + "\n", + " with ThreadPoolExecutor(max_workers=max_workers) as ex:\n", + " futures = {ex.submit(download_one_variable, var, args): var for var in variables}\n", + " for fut in as_completed(futures):\n", + " var, ok, fail = fut.result()\n", + " results.append((var, ok, fail))\n", + " total_ok += ok\n", + " total_fail += fail\n", + " print(f\"[DONE][{var}] Ok={ok}, Fail={fail}\")\n", + "\n", + " elapsed = time.time() - start\n", + " print(\"\\n================ SUMMARY ================\")\n", + " for var, ok, fail in sorted(results):\n", + " print(f\"{var:40s} Ok={ok:4d} Fail={fail:3d}\")\n", + " print(f\"-----------------------------------------\")\n", + " print(f\"TOTAL Ok={total_ok} Fail={total_fail} | Elapsed: {elapsed/60:.1f} min\")\n", + " print(\"=========================================\\n\")\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 854, + "referenced_widgets": [ + "77f152bb34f64833ad4f9e10337992f6", + "79cf5f4c819246149f0b3b7cfaf03b5b", + "c142db7f1d6c4222b3811342b46b0c30", + "485aa27dcfbb4cb9a61da98a8ad277c1", + "652a7f72be404e2580fb4592348330cb", + "76fd9553926040f5a108afe18be359b8", + "492b0421e1cc4ee2b02d24910fbbd367", + "4065f75702ea4b30b25b3e0adc6d1cf4", + "8d4b92424bde4065b92126463c3a88cf", + "da3c858f56284927907e863b3f7bff15", + "17a5e81074e042208e7c41a6fe72d705", + "7638bda1ed3e4c66a4a47c3ff6a8edea", + "b30de5c069a4404ab2f48af8feef79cc", + "23894ce319a141d19310e4c02124bd38", + "2bf99ea9822d41d39a8fbd324bb8b3e5", + "3a5456e2d86a48be853f4552be673819", + "2f8e3c66060b4ef1b57aa2ea97259b7b", + "daf5ce1cd8e44d47aa090b2693b83d10", + "701ad51f4d414a3fbaa9c9eb5e448785", + "5b81063a9d2f46d69a3d138704717817", + "87ca252dfca34361ab50ea1809856703", + "db01f8756847498482428b5460105a6b" + ] + }, + "id": "2acnD1HPz3l5", + "outputId": "45f323f6-0ec7-4c0f-910c-dd0ed415e62e" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[INFO] Variables: 54 | Years: 26 | Months: 12\n", + "[INFO] Parallel by VARIABLE with max_workers = 3\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:12:52,047 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:12:52,049 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:12:52,245 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:12:52,247 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:12:52,250 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[INFO][skin_temperature] Downloading 1997-01 -> era5land/skin_temperature/1997/reanalysis-era5-land_skin_temperature_1997-01.grib.zip\n", + "[INFO][2m_temperature] Downloading 1997-01 -> era5land/2m_temperature/1997/reanalysis-era5-land_2m_temperature_1997-01.grib.zip\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:12:52,255 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[INFO][2m_dewpoint_temperature] Downloading 1997-01 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-01.grib.zip\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:12:53,041 INFO Request ID is aebbba26-5aa4-4aca-b888-df4bc5037557\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is aebbba26-5aa4-4aca-b888-df4bc5037557\n", + "2025-09-28 10:12:53,083 INFO Request ID is 1136a560-fa8e-4f6c-863b-c2eb22028995\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 1136a560-fa8e-4f6c-863b-c2eb22028995\n", + "2025-09-28 10:12:53,255 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:12:53,327 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:12:53,394 INFO Request ID is 3ccd2bcf-f2c5-4875-bbfb-33c845fd886b\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 3ccd2bcf-f2c5-4875-bbfb-33c845fd886b\n", + "2025-09-28 10:12:53,563 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:13:07,327 INFO status has been updated to successful\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to successful\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "e5dc627d8a097bec79906e75846db42e.zip: 0%| | 0.00/4.46M [00:00 era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-02.grib.zip\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:13:11,084 INFO Request ID is c96d1a54-2476-40f3-ba1c-e360b6529ef7\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is c96d1a54-2476-40f3-ba1c-e360b6529ef7\n", + "2025-09-28 10:13:11,231 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:13:15,343 INFO status has been updated to running\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to running\n", + "2025-09-28 10:14:48,504 INFO status has been updated to successful\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to successful\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "76d9dacc7921e0bd06d9383480d75e62.zip: 0%| | 0.00/4.47M [00:00 era5land/2m_temperature/1997/reanalysis-era5-land_2m_temperature_1997-02.grib.zip\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:14:51,964 INFO Request ID is f4c1333f-a60a-47f8-84fc-e9e2d4555026\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is f4c1333f-a60a-47f8-84fc-e9e2d4555026\n", + "2025-09-28 10:14:52,108 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# download_era5land_chunked_full_month_parallel.py —— 按“月份”并发（每年最多12并发）\n", + "# -*- coding: utf-8 -*-\n", + "import argparse\n", + "import time\n", + "import random\n", + "from pathlib import Path\n", + "from typing import List, Tuple\n", + "import sys\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "\n", + "import cdsapi\n", + "\n", + "DATASET = \"reanalysis-era5-land\"\n", + "\n", + "# ---------- 时间维度 ----------\n", + "ALL_DAYS: List[str] = [f\"{d:02d}\" for d in range(1, 32)]\n", + "ALL_HOURS: List[str] = [f\"{h:02d}:00\" for h in range(0, 24)]\n", + "ALL_MONTHS: List[str] = [f\"{m:02d}\" for m in range(1, 13)]\n", + "\n", + "# ---------- 重试设置 ----------\n", + "MAX_RETRIES = 8\n", + "BASE_SLEEP = 10 # seconds\n", + "\n", + "# ---------- 变量全集（你的清单 + 注释项全部纳入） ----------\n", + "VARIABLES: List[str] = [\n", + " # 2m/skin/soil/lake temps\n", + " \"2m_dewpoint_temperature\",\n", + " \"2m_temperature\",\n", + " \"skin_temperature\",\n", + " \"soil_temperature_level_1\",\n", + " \"soil_temperature_level_2\",\n", + " \"soil_temperature_level_3\",\n", + " \"soil_temperature_level_4\",\n", + " \"lake_bottom_temperature\",\n", + " \"lake_ice_depth\",\n", + " \"lake_ice_temperature\",\n", + " \"lake_mix_layer_depth\",\n", + " \"lake_mix_layer_temperature\",\n", + " \"lake_shape_factor\",\n", + " # 你原注释掉的项（已启用）\n", + " \"lake_total_layer_temperature\",\n", + " \"snow_albedo\",\n", + " \"snow_cover\",\n", + " \"snow_density\",\n", + " \"snow_depth\",\n", + " \"snow_depth_water_equivalent\",\n", + " \"snowfall\",\n", + " \"snowmelt\",\n", + " \"temperature_of_snow_layer\",\n", + " \"forecast_albedo\",\n", + " \"surface_latent_heat_flux\",\n", + " \"surface_net_solar_radiation\",\n", + " \"surface_net_thermal_radiation\",\n", + " \"surface_sensible_heat_flux\",\n", + " \"surface_solar_radiation_downwards\",\n", + " \"surface_thermal_radiation_downwards\",\n", + " \"evaporation_from_bare_soil\",\n", + " \"evaporation_from_open_water_surfaces_excluding_oceans\",\n", + " \"evaporation_from_the_top_of_canopy\",\n", + " \"evaporation_from_vegetation_transpiration\",\n", + " \"potential_evaporation\",\n", + " \"runoff\",\n", + " \"snow_evaporation\",\n", + " \"sub_surface_runoff\",\n", + " \"surface_runoff\",\n", + " \"total_evaporation\",\n", + " \"10m_u_component_of_wind\",\n", + " \"10m_v_component_of_wind\",\n", + " \"surface_pressure\",\n", + " \"total_precipitation\",\n", + " \"leaf_area_index_high_vegetation\",\n", + " \"leaf_area_index_low_vegetation\",\n", + " \"high_vegetation_cover\",\n", + " \"glacier_mask\",\n", + " \"lake_cover\",\n", + " \"low_vegetation_cover\",\n", + " \"lake_total_depth\",\n", + " \"land_sea_mask\",\n", + " \"soil_type\",\n", + " \"type_of_high_vegetation\",\n", + " \"type_of_low_vegetation\",\n", + "]\n", + "\n", + "def build_request(\n", + " variable: str,\n", + " year: str,\n", + " month: str,\n", + " area_box: Tuple[float, float, float, float],\n", + " fmt: str,\n", + ") -> dict:\n", + " north, west, south, east = area_box\n", + " req = {\n", + " \"variable\": variable,\n", + " \"year\": year,\n", + " \"month\": month,\n", + " \"day\": ALL_DAYS,\n", + " \"time\": ALL_HOURS,\n", + " \"area\": [north, west, south, east], # N W S E\n", + " \"format\": fmt, # \"grib\" | \"netcdf\"\n", + " \"download_format\": \"zip\",\n", + " # \"product_type\": \"reanalysis\",\n", + " }\n", + " return req\n", + "\n", + "def safe_retrieve(client: cdsapi.Client, dataset: str, request: dict, target_path: Path):\n", + " attempt = 0\n", + " # 轻微抖动，错峰请求\n", + " time.sleep(random.uniform(0.3, 1.0))\n", + " while True:\n", + " try:\n", + " client.retrieve(dataset, request).download(str(target_path))\n", + " return True\n", + " except Exception as e:\n", + " attempt += 1\n", + " msg = str(e).lower()\n", + " # 明确不可恢复的错误（变量无效/不可用/无数据）直接跳过\n", + " unrecoverable_signals = [\n", + " \"unavailable\",\n", + " \"not available\",\n", + " \"invalid\",\n", + " \"does not match\",\n", + " \"no data\",\n", + " \"bad request\",\n", + " \"cannot be found\",\n", + " ]\n", + " if any(s in msg for s in unrecoverable_signals):\n", + " print(f\"[ERROR] Unrecoverable for {target_path.name}: {e}\")\n", + " return False\n", + "\n", + " if attempt > MAX_RETRIES:\n", + " print(f\"[ERROR] Max retries exceeded for {target_path.name}: {e}\")\n", + " return False\n", + "\n", + " sleep_s = BASE_SLEEP * (2 ** (attempt - 1)) * random.uniform(0.85, 1.15)\n", + " print(f\"[WARN] Download failed (attempt {attempt}/{MAX_RETRIES}): {e}\")\n", + " print(f\" Sleeping {sleep_s:.0f}s then retrying...\")\n", + " time.sleep(sleep_s)\n", + "\n", + "def parse_args_with_defaults():\n", + " parser = argparse.ArgumentParser(\n", + " description=\"ERA5-Land downloader (split by variable × year × month) — month-level parallelism\"\n", + " )\n", + " # —— 给出默认值，不再强制要求 —— #\n", + " parser.add_argument(\"--out_dir\", type=str, default=\"./era5land\",\n", + " help=\"输出根目录（默认 ./era5land）\")\n", + " parser.add_argument(\"--bbox\", nargs=4, type=float,\n", + " default=[60.86, -6.23, 49.86, 1.75],\n", + " metavar=(\"NORTH\", \"WEST\", \"SOUTH\", \"EAST\"),\n", + " help=\"经纬度范围：N W S E（默认 60.86 -6.23 49.86 1.75）\")\n", + " parser.add_argument(\"--format\", default=\"grib\", choices=[\"grib\", \"netcdf\"],\n", + " help=\"文件格式（默认 grib）\")\n", + " parser.add_argument(\"--years\", nargs=\"+\",\n", + " default=[str(y) for y in range(1997, 2023)], # 1997–2022\n", + " help=\"年份列表（默认 1997..2022）\")\n", + " parser.add_argument(\"--months\", nargs=\"+\", default=ALL_MONTHS,\n", + " help=\"月份列表（默认 01..12）\")\n", + " parser.add_argument(\"--variables\", nargs=\"+\", default=VARIABLES,\n", + " help=\"变量名列表（默认为脚本内置全集）\")\n", + " parser.add_argument(\"--skip_existing\", action=\"store_true\",\n", + " help=\"若目标文件已存在则跳过\")\n", + " parser.add_argument(\"--month_workers\", type=int, default=12,\n", + " help=\"每个 年×变量 的月份并发数（默认 12）\")\n", + " # 如果在 Notebook 中直接运行，且没有传任何参数，也能用默认值\n", + " try:\n", + " return parser.parse_args([])\n", + " except SystemExit:\n", + " # 在某些环境 parse_args([]) 会触发 SystemExit，退回到标准方式\n", + " return parser.parse_args()\n", + "\n", + "def download_one_month(var: str, year: str, month: str, args) -> bool:\n", + " \"\"\"并发任务：下载单个 变量×年×月 分块\"\"\"\n", + " subdir = Path(args.out_dir) / var / str(year)\n", + " subdir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " suffix = \"grib\" if args.format == \"grib\" else \"nc\"\n", + " target_name = f\"{DATASET}_{var}_{year}-{month}.{suffix}.zip\"\n", + " target_path = subdir / target_name\n", + "\n", + " if args.skip_existing and target_path.exists():\n", + " # 已存在直接视为成功（简易断点续跑）\n", + " return True\n", + "\n", + " req = build_request(\n", + " variable=var,\n", + " year=str(year),\n", + " month=f\"{int(month):02d}\",\n", + " area_box=tuple(args.bbox),\n", + " fmt=args.format,\n", + " )\n", + "\n", + " print(f\"[INFO][{var}][{year}] Downloading month={month} -> {target_path}\")\n", + " # 为了线程安全，这里每个任务各自实例化 client\n", + " client = cdsapi.Client()\n", + " return safe_retrieve(client, DATASET, req, target_path)\n", + "\n", + "def main():\n", + " # 在 Notebook/Colab 里，这里会采用默认值；命令行下可用参数覆盖\n", + " if \"ipykernel\" in sys.modules or \"google.colab\" in sys.modules:\n", + " args = parse_args_with_defaults()\n", + " else:\n", + " args = parse_args_with_defaults()\n", + "\n", + " total_ok = 0\n", + " total_fail = 0\n", + "\n", + " for var in args.variables:\n", + " for year in args.years:\n", + " months = list(args.months)\n", + " max_workers = max(1, min(args.month_workers, len(months)))\n", + " print(f\"\\n[GROUP] var={var} year={year} | months={months} | month_workers={max_workers}\")\n", + "\n", + " futures = []\n", + " with ThreadPoolExecutor(max_workers=max_workers) as ex:\n", + " for month in months:\n", + " futures.append(ex.submit(download_one_month, var, year, month, args))\n", + " for fut in as_completed(futures):\n", + " ok = fut.result()\n", + " if ok: total_ok += 1\n", + " else: total_fail += 1\n", + "\n", + " print(f\"\\n[DONE] Finished. Success: {total_ok}, Failed: {total_fail}\")\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EQN9ggBX0U6D", + "outputId": "58899f9d-5504-4a22-8b80-65b4f9f3faab" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\n", + "[GROUP] var=2m_dewpoint_temperature year=1997 | months=['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'] | month_workers=12\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=01 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-01.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=02 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-02.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=03 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-03.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=05 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-05.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=04 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-04.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=06 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-06.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=07 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-07.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=09 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-09.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=08 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-08.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=10 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-10.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=11 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-11.grib.zip\n", + "[INFO][2m_dewpoint_temperature][1997] Downloading month=12 -> era5land/2m_dewpoint_temperature/1997/reanalysis-era5-land_2m_dewpoint_temperature_1997-12.grib.zip\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-09-28 10:16:14,916 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,917 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,919 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,920 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,926 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,933 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,935 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,939 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,942 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,935 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,936 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,936 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,945 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,953 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,948 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,956 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,949 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,963 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,964 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,970 INFO [2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,973 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,975 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:14,974 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2025-09-03T00:00:00] To improve our C3S service, we need to hear from you! Please complete this very short [survey](https://confluence.ecmwf.int/x/E7uBEQ/). Thank you.\n", + "2025-09-28 10:16:14,979 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "INFO:ecmwf.datastores.legacy_client:[2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n", + "2025-09-28 10:16:15,818 INFO Request ID is 067dc222-585a-4744-a0ac-4570fda17814\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 067dc222-585a-4744-a0ac-4570fda17814\n", + "2025-09-28 10:16:15,831 INFO Request ID is 5f47f2e0-1aa4-47ae-829f-b05b53c69363\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 5f47f2e0-1aa4-47ae-829f-b05b53c69363\n", + "2025-09-28 10:16:15,848 INFO Request ID is a9be8a75-74e5-49d4-9441-f4f653fc3f6b\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is a9be8a75-74e5-49d4-9441-f4f653fc3f6b\n", + "2025-09-28 10:16:15,857 INFO Request ID is ec12ac64-6a87-475e-9853-1a5c7bef1419\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is ec12ac64-6a87-475e-9853-1a5c7bef1419\n", + "2025-09-28 10:16:15,941 INFO Request ID is ae343a40-f746-4efb-97e7-9e4384ab6aa2\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is ae343a40-f746-4efb-97e7-9e4384ab6aa2\n", + "2025-09-28 10:16:15,979 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:15,982 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:15,991 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:15,991 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:15,993 INFO Request ID is 85e7d622-6f8b-4f36-be77-c728883e832b\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 85e7d622-6f8b-4f36-be77-c728883e832b\n", + "2025-09-28 10:16:16,060 INFO Request ID is ac540eec-458d-4517-b2aa-7fd0db3a88c7\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is ac540eec-458d-4517-b2aa-7fd0db3a88c7\n", + "2025-09-28 10:16:16,079 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:16,097 INFO Request ID is 5c27fe51-87b7-4304-9a1c-29dbe883e1af\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 5c27fe51-87b7-4304-9a1c-29dbe883e1af\n", + "2025-09-28 10:16:16,098 INFO Request ID is 9101737b-5c7b-4c01-b771-ca3bb6180867\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is 9101737b-5c7b-4c01-b771-ca3bb6180867\n", + "2025-09-28 10:16:16,132 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:16,228 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:16,234 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:16,288 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "WARNING:multiurl.http:Recovering from HTTP error [429 Too Many Requests], attempt 1 of 500\n", + "WARNING:multiurl.http:Retrying in 120 seconds\n", + "2025-09-28 10:16:16,332 INFO Request ID is f150c332-0ddd-47bb-b448-fb25fa21d8b7\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is f150c332-0ddd-47bb-b448-fb25fa21d8b7\n", + "2025-09-28 10:16:16,354 INFO Request ID is e8268ff6-f4b4-4aac-927a-5137a23ac5ad\n", + "INFO:ecmwf.datastores.legacy_client:Request ID is e8268ff6-f4b4-4aac-927a-5137a23ac5ad\n", + "2025-09-28 10:16:16,465 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n", + "2025-09-28 10:16:16,498 INFO status has been updated to accepted\n", + "INFO:ecmwf.datastores.legacy_client:status has been updated to accepted\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/jointContribution/AI_Climate_Diseases/imputer.ipynb b/jointContribution/AI_Climate_Diseases/imputer.ipynb new file mode 100644 index 0000000000..3a4a3cc565 --- /dev/null +++ b/jointContribution/AI_Climate_Diseases/imputer.ipynb @@ -0,0 +1,629 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rlAsSGECAmNA", + "outputId": "76610133-ea9d-4566-b7fe-bc3340db872d" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[OK] Ownsend_Deprivation_Index: {'column': 'Ownsend_Deprivation_Index', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.03671', 'metric_secondary': 'r2=0.6825', 'fallback': 'none'}\n", + "[OK] Number_of_Self-Reported_Cancers: {'column': 'Number_of_Self-Reported_Cancers', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01536', 'metric_secondary': 'r2=0.8008', 'fallback': 'none'}\n", + "[OK] Operations: {'column': 'Operations', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.65087', 'metric_secondary': 'r2=0.3075', 'fallback': 'none'}\n", + "[OK] Number_of_Treatments/Medications: {'column': 'Number_of_Treatments/Medications', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2.40922', 'metric_secondary': 'r2=0.6692', 'fallback': 'none'}\n", + "[OK] Number_of_Self-Reported_Non-Cancer_Illnesses: {'column': 'Number_of_Self-Reported_Non-Cancer_Illnesses', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.53121', 'metric_secondary': 'r2=0.5638', 'fallback': 'none'}\n", + "[OK] Aidememoire_Completed: {'column': 'Aidememoire_Completed', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.15893', 'metric_secondary': 'r2=0.0430', 'fallback': 'median'}\n", + "[OK] Sexual_History: {'column': 'Sexual_History', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.05077', 'metric_secondary': 'r2=0.4014', 'fallback': 'none'}\n", + "[OK] Added_Salt: {'column': 'Added_Salt', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.22884', 'metric_secondary': 'r2=0.0754', 'fallback': 'none'}\n", + "[OK] Handedness: {'column': 'Handedness', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09320', 'metric_secondary': 'r2=0.0545', 'fallback': 'none'}\n", + "[OK] Current_Tobacco_Smoking: {'column': 'Current_Tobacco_Smoking', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00001', 'metric_secondary': 'r2=0.9999', 'fallback': 'none'}\n", + "[OK] Accommodation_Type: {'column': 'Accommodation_Type', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.05787', 'metric_secondary': 'r2=0.4082', 'fallback': 'none'}\n", + "[OK] Alcohol_Intake_Frequency: {'column': 'Alcohol_Intake_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00034', 'metric_secondary': 'r2=0.9956', 'fallback': 'none'}\n", + "[OK] Milk_Type: {'column': 'Milk_Type', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19374', 'metric_secondary': 'r2=0.1529', 'fallback': 'none'}\n", + "[OK] Insomnia: {'column': 'Insomnia', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15658', 'metric_secondary': 'r2=0.1454', 'fallback': 'none'}\n", + "[OK] Glasses_Wear: {'column': 'Glasses_Wear', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08702', 'metric_secondary': 'r2=0.1445', 'fallback': 'none'}\n", + "[OK] Alcohol_Status: {'column': 'Alcohol_Status', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00001', 'metric_secondary': 'r2=0.9999', 'fallback': 'none'}\n", + "[OK] Pacemaker: {'column': 'Pacemaker', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00194', 'metric_secondary': 'r2=0.3678', 'fallback': 'none'}\n", + "[OK] Computer_Gaming: {'column': 'Computer_Gaming', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.14589', 'metric_secondary': 'r2=0.1162', 'fallback': 'none'}\n", + "[OK] Birth_Country: {'column': 'Birth_Country', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08208', 'metric_secondary': 'r2=0.5226', 'fallback': 'none'}\n", + "[OK] Day_Napping: {'column': 'Day_Napping', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20236', 'metric_secondary': 'r2=0.1787', 'fallback': 'none'}\n", + "[OK] Hair_Color: {'column': 'Hair_Color', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.22422', 'metric_secondary': 'r2=0.0561', 'fallback': 'none'}\n", + "[OK] Poultry: {'column': 'Poultry', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01650', 'metric_secondary': 'r2=0.6699', 'fallback': 'none'}\n", + "[OK] Waist_Circumference: {'column': 'Waist_Circumference', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=21.89398', 'metric_secondary': 'r2=0.8808', 'fallback': 'none'}\n", + "[OK] Tea: {'column': 'Tea', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=6.09476', 'metric_secondary': 'r2=0.2554', 'fallback': 'none'}\n", + "[OK] Hip_Circumference: {'column': 'Hip_Circumference', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=13.19761', 'metric_secondary': 'r2=0.8465', 'fallback': 'none'}\n", + "[OK] Processed_Meat_Intake: {'column': 'Processed_Meat_Intake', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04308', 'metric_secondary': 'r2=0.5001', 'fallback': 'none'}\n", + "[OK] Coffee: {'column': 'Coffee', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.11790', 'metric_secondary': 'r2=0.2826', 'fallback': 'none'}\n", + "[OK] Diet_Change: {'column': 'Diet_Change', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20395', 'metric_secondary': 'r2=0.1449', 'fallback': 'none'}\n", + "[OK] Adopted: {'column': 'Adopted', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.01467', 'metric_secondary': 'r2=-0.0032', 'fallback': 'median'}\n", + "[OK] Standing_Height: {'column': 'Standing_Height', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.92470', 'metric_secondary': 'r2=0.9892', 'fallback': 'none'}\n", + "[OK] Diabetes_Diagnosis: {'column': 'Diabetes_Diagnosis', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01807', 'metric_secondary': 'r2=0.6474', 'fallback': 'none'}\n", + "[OK] Eye_Problems: {'column': 'Eye_Problems', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11966', 'metric_secondary': 'r2=0.0556', 'fallback': 'none'}\n", + "[OK] Falls: {'column': 'Falls', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.14412', 'metric_secondary': 'r2=0.0938', 'fallback': 'none'}\n", + "[OK] Current_Residence_Duration: {'column': 'Current_Residence_Duration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=111.30535', 'metric_secondary': 'r2=0.2432', 'fallback': 'none'}\n", + "[OK] Cancer_Diagnosis: {'column': 'Cancer_Diagnosis', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01289', 'metric_secondary': 'r2=0.8208', 'fallback': 'none'}\n", + "[OK] Weight: {'column': 'Weight', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.38324', 'metric_secondary': 'r2=0.9985', 'fallback': 'none'}\n", + "[OK] Ethnicity: {'column': 'Ethnicity', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.05108', 'metric_secondary': 'r2=0.4994', 'fallback': 'none'}\n", + "[OK] Smoked: {'column': 'Smoked', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19598', 'metric_secondary': 'r2=0.1850', 'fallback': 'none'}\n", + "[OK] Smoking_Status: {'column': 'Smoking_Status', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00003', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Other_Prescription_Medications: {'column': 'Other_Prescription_Medications', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11891', 'metric_secondary': 'r2=0.5227', 'fallback': 'none'}\n", + "[OK] BMI: {'column': 'BMI', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.06996', 'metric_secondary': 'r2=0.9969', 'fallback': 'none'}\n", + "[OK] Cereal: {'column': 'Cereal', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=6.16983', 'metric_secondary': 'r2=0.2266', 'fallback': 'none'}\n", + "[OK] Fresh_Fruit: {'column': 'Fresh_Fruit', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2.10727', 'metric_secondary': 'r2=0.2082', 'fallback': 'none'}\n", + "[OK] Hand_Grip_Strength_(Right): {'column': 'Hand_Grip_Strength_(Right)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=20.45542', 'metric_secondary': 'r2=0.8401', 'fallback': 'none'}\n", + "[OK] Beef_Intake: {'column': 'Beef_Intake', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04843', 'metric_secondary': 'r2=0.5084', 'fallback': 'none'}\n", + "[OK] Hand_Grip_Strength_(Left): {'column': 'Hand_Grip_Strength_(Left)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=20.41612', 'metric_secondary': 'r2=0.8406', 'fallback': 'none'}\n", + "[OK] Overall_Health_Rating: {'column': 'Overall_Health_Rating', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02892', 'metric_secondary': 'r2=0.3373', 'fallback': 'none'}\n", + "[OK] Psychiatrist_Visits: {'column': 'Psychiatrist_Visits', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07183', 'metric_secondary': 'r2=0.2951', 'fallback': 'none'}\n", + "[OK] Non_Oily_Fish: {'column': 'Non_Oily_Fish', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02499', 'metric_secondary': 'r2=0.4516', 'fallback': 'none'}\n", + "[OK] Fractures: {'column': 'Fractures', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.08456', 'metric_secondary': 'r2=0.0343', 'fallback': 'median'}\n", + "[OK] Daytime_Dozing: {'column': 'Daytime_Dozing', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15195', 'metric_secondary': 'r2=0.1684', 'fallback': 'none'}\n", + "[OK] Spirometry_Contraindications: {'column': 'Spirometry_Contraindications', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00493', 'metric_secondary': 'r2=0.9302', 'fallback': 'none'}\n", + "[OK] Oily_Fish: {'column': 'Oily_Fish', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.06648', 'metric_secondary': 'r2=0.3064', 'fallback': 'none'}\n", + "[OK] Sleep_Duration: {'column': 'Sleep_Duration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.14679', 'metric_secondary': 'r2=0.0885', 'fallback': 'none'}\n", + "[OK] Pork: {'column': 'Pork', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08622', 'metric_secondary': 'r2=0.4008', 'fallback': 'none'}\n", + "[OK] Lamb_Mutton: {'column': 'Lamb_Mutton', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09012', 'metric_secondary': 'r2=0.3838', 'fallback': 'none'}\n", + "[OK] Incorrect_Matches: {'column': 'Incorrect_Matches', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.19447', 'metric_secondary': 'r2=0.1079', 'fallback': 'none'}\n", + "[OK] Willingness_for_Cognitive_Tests: {'column': 'Willingness_for_Cognitive_Tests', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00715', 'metric_secondary': 'r2=0.6377', 'fallback': 'none'}\n", + "[OK] Water_Intake: {'column': 'Water_Intake', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4.02436', 'metric_secondary': 'r2=0.2531', 'fallback': 'none'}\n", + "[OK] Water_300m: {'column': 'Water_300m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=82.21683', 'metric_secondary': 'r2=0.8698', 'fallback': 'none'}\n", + "[OK] Natural_Environment_(1000m_Buffer): {'column': 'Natural_Environment_(1000m_Buffer)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=20.10810', 'metric_secondary': 'r2=0.9693', 'fallback': 'none'}\n", + "[OK] Natural_Environment_300m: {'column': 'Natural_Environment_300m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=24.21351', 'metric_secondary': 'r2=0.9687', 'fallback': 'none'}\n", + "[OK] GP_Visits_for_Mental_Health: {'column': 'GP_Visits_for_Mental_Health', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13707', 'metric_secondary': 'r2=0.3873', 'fallback': 'none'}\n", + "[OK] Home_Area_Population_Density: {'column': 'Home_Area_Population_Density', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01900', 'metric_secondary': 'r2=0.8828', 'fallback': 'none'}\n", + "[OK] TV_Time: {'column': 'TV_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2.19738', 'metric_secondary': 'r2=0.2620', 'fallback': 'none'}\n", + "[OK] Disability/Mobility_Allowance: {'column': 'Disability/Mobility_Allowance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03385', 'metric_secondary': 'r2=0.4068', 'fallback': 'none'}\n", + "[OK] Wake_Up_Ease: {'column': 'Wake_Up_Ease', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03346', 'metric_secondary': 'r2=0.1082', 'fallback': 'none'}\n", + "[OK] Spread_Type: {'column': 'Spread_Type', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08664', 'metric_secondary': 'r2=0.1012', 'fallback': 'none'}\n", + "[OK] Match_Identification_Time: {'column': 'Match_Identification_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=11765.66602', 'metric_secondary': 'r2=0.1531', 'fallback': 'none'}\n", + "[OK] UV_Protection: {'column': 'UV_Protection', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07551', 'metric_secondary': 'r2=0.1674', 'fallback': 'none'}\n", + "[OK] Seated_Height: {'column': 'Seated_Height', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=9.22653', 'metric_secondary': 'r2=0.8201', 'fallback': 'none'}\n", + "[OK] Seating_Box_Height: {'column': 'Seating_Box_Height', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00028', 'metric_secondary': 'r2=0.8925', 'fallback': 'none'}\n", + "[OK] Sitting_Height: {'column': 'Sitting_Height', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.85534', 'metric_secondary': 'r2=0.9227', 'fallback': 'none'}\n", + "[OK] Hot_Drink_Temperature: {'column': 'Hot_Drink_Temperature', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.12885', 'metric_secondary': 'r2=0.0284', 'fallback': 'median'}\n", + "[OK] Chest_Pain_Discomfort: {'column': 'Chest_Pain_Discomfort', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11280', 'metric_secondary': 'r2=0.1704', 'fallback': 'none'}\n", + "[OK] Dried_Fruit: {'column': 'Dried_Fruit', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2.92822', 'metric_secondary': 'r2=0.0881', 'fallback': 'none'}\n", + "[OK] Diet_Variation: {'column': 'Diet_Variation', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20759', 'metric_secondary': 'r2=0.0766', 'fallback': 'none'}\n", + "[OK] Major_Road_Traffic: {'column': 'Major_Road_Traffic', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00004', 'metric_secondary': 'r2=0.8855', 'fallback': 'none'}\n", + "[OK] Nearest_Road_Distance: {'column': 'Nearest_Road_Distance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=49187815.57922', 'metric_secondary': 'r2=0.8913', 'fallback': 'none'}\n", + "[OK] PM10_Air_2007: {'column': 'PM10_Air_2007', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00837', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Nearest_Road_Traffic: {'column': 'Nearest_Road_Traffic', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00144', 'metric_secondary': 'r2=0.7221', 'fallback': 'none'}\n", + "[OK] NO2_Air_Pollution_(2005): {'column': 'NO2_Air_Pollution_(2005)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.45550', 'metric_secondary': 'r2=0.9955', 'fallback': 'none'}\n", + "[OK] NO2_Air_Pollution_(2006): {'column': 'NO2_Air_Pollution_(2006)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.37226', 'metric_secondary': 'r2=0.9956', 'fallback': 'none'}\n", + "[OK] PM2.5_to_10_Air_2010: {'column': 'PM2.5_to_10_Air_2010', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4921075.06406', 'metric_secondary': 'r2=0.7997', 'fallback': 'none'}\n", + "[OK] Major_Road_Distance: {'column': 'Major_Road_Distance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=89375062275.11403', 'metric_secondary': 'r2=0.9296', 'fallback': 'none'}\n", + "[OK] NO2_Air_Pollution_(2010): {'column': 'NO2_Air_Pollution_(2010)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.68771', 'metric_secondary': 'r2=0.9879', 'fallback': 'none'}\n", + "[OK] Road_Traffic_Load: {'column': 'Road_Traffic_Load', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=558.05541', 'metric_secondary': 'r2=0.9083', 'fallback': 'none'}\n", + "[OK] Trunk_Fat_Percentage: {'column': 'Trunk_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=11.20265', 'metric_secondary': 'r2=0.9531', 'fallback': 'none'}\n", + "[OK] Evening_Noise_Level: {'column': 'Evening_Noise_Level', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00841', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Nighttime_Noise_Level: {'column': 'Nighttime_Noise_Level', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00860', 'metric_secondary': 'r2=0.9995', 'fallback': 'none'}\n", + "[OK] Proximity_to_Major_Road: {'column': 'Proximity_to_Major_Road', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00491', 'metric_secondary': 'r2=0.9277', 'fallback': 'none'}\n", + "[OK] Hour-16_Noise_Level: {'column': 'Hour-16_Noise_Level', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00856', 'metric_secondary': 'r2=0.9995', 'fallback': 'none'}\n", + "[OK] Daytime_Noise_Level: {'column': 'Daytime_Noise_Level', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00831', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Road_Length: {'column': 'Road_Length', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.47892', 'metric_secondary': 'r2=0.9958', 'fallback': 'none'}\n", + "[OK] Cooked_Vegetables: {'column': 'Cooked_Vegetables', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.27094', 'metric_secondary': 'r2=0.1751', 'fallback': 'none'}\n", + "[OK] Salad_Intake: {'column': 'Salad_Intake', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.73412', 'metric_secondary': 'r2=0.2083', 'fallback': 'none'}\n", + "[OK] Social_Visits: {'column': 'Social_Visits', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01657', 'metric_secondary': 'r2=0.0511', 'fallback': 'none'}\n", + "[OK] Recent_Stress_Events: {'column': 'Recent_Stress_Events', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.22579', 'metric_secondary': 'r2=0.0891', 'fallback': 'none'}\n", + "[OK] Phone_Use_Duration: {'column': 'Phone_Use_Duration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11437', 'metric_secondary': 'r2=0.1156', 'fallback': 'none'}\n", + "[OK] Skin_Color: {'column': 'Skin_Color', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19061', 'metric_secondary': 'r2=0.1207', 'fallback': 'none'}\n", + "[OK] NO2_Air_2007: {'column': 'NO2_Air_2007', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.47331', 'metric_secondary': 'r2=0.9427', 'fallback': 'none'}\n", + "[OK] Computer_Time: {'column': 'Computer_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.83010', 'metric_secondary': 'r2=0.1338', 'fallback': 'none'}\n", + "[OK] Bowel_Screening: {'column': 'Bowel_Screening', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17538', 'metric_secondary': 'r2=0.1949', 'fallback': 'none'}\n", + "[OK] Weight_Change_(1_Year): {'column': 'Weight_Change_(1_Year)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.21421', 'metric_secondary': 'r2=0.1316', 'fallback': 'none'}\n", + "[OK] Loneliness/Isolation: {'column': 'Loneliness/Isolation', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.10941', 'metric_secondary': 'r2=0.2875', 'fallback': 'none'}\n", + "[OK] Driving_Time: {'column': 'Driving_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.24548', 'metric_secondary': 'r2=0.2583', 'fallback': 'none'}\n", + "[OK] Whole_Body_Water_Mass: {'column': 'Whole_Body_Water_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02345', 'metric_secondary': 'r2=0.9997', 'fallback': 'none'}\n", + "[OK] Basal_Metabolic_Rate: {'column': 'Basal_Metabolic_Rate', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=469.91901', 'metric_secondary': 'r2=0.9997', 'fallback': 'none'}\n", + "[OK] Right_Leg_Impedance: {'column': 'Right_Leg_Impedance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=21.39709', 'metric_secondary': 'r2=0.9839', 'fallback': 'none'}\n", + "[OK] Left_Leg_Impedance: {'column': 'Left_Leg_Impedance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=18.76451', 'metric_secondary': 'r2=0.9853', 'fallback': 'none'}\n", + "[OK] Whole_Body_Fat-Free_Mass: {'column': 'Whole_Body_Fat-Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02022', 'metric_secondary': 'r2=0.9998', 'fallback': 'none'}\n", + "[OK] Right_Leg_Fat_Percentage: {'column': 'Right_Leg_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07534', 'metric_secondary': 'r2=0.9993', 'fallback': 'none'}\n", + "[OK] Left_Arm_Impedance: {'column': 'Left_Arm_Impedance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=30.00962', 'metric_secondary': 'r2=0.9907', 'fallback': 'none'}\n", + "[OK] Right_Leg_Fat_Mass: {'column': 'Right_Leg_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00680', 'metric_secondary': 'r2=0.9981', 'fallback': 'none'}\n", + "[OK] Right_Leg_Fat_Free_Mass: {'column': 'Right_Leg_Fat_Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00192', 'metric_secondary': 'r2=0.9995', 'fallback': 'none'}\n", + "[OK] Right_Leg_Predicted_Mass: {'column': 'Right_Leg_Predicted_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00188', 'metric_secondary': 'r2=0.9995', 'fallback': 'none'}\n", + "[OK] Whole_Body_Impedance: {'column': 'Whole_Body_Impedance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=157.82494', 'metric_secondary': 'r2=0.9803', 'fallback': 'none'}\n", + "[OK] Left_Leg_Fat_Percentage: {'column': 'Left_Leg_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07565', 'metric_secondary': 'r2=0.9993', 'fallback': 'none'}\n", + "[OK] Right_Arm_Impedance: {'column': 'Right_Arm_Impedance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=85.94955', 'metric_secondary': 'r2=0.9723', 'fallback': 'none'}\n", + "[OK] Left_Leg_Fat_Mass: {'column': 'Left_Leg_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00534', 'metric_secondary': 'r2=0.9985', 'fallback': 'none'}\n", + "[OK] Left_Leg_Fat-Free_Mass: {'column': 'Left_Leg_Fat-Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00176', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Left_Leg_Predicted_Mass: {'column': 'Left_Leg_Predicted_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00150', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Right_Arm_Fat_Percentage: {'column': 'Right_Arm_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16739', 'metric_secondary': 'r2=0.9984', 'fallback': 'none'}\n", + "[OK] Right_Arm_Fat_Mass: {'column': 'Right_Arm_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00189', 'metric_secondary': 'r2=0.9954', 'fallback': 'none'}\n", + "[OK] Right_Arm_Fat-Free_Mass: {'column': 'Right_Arm_Fat-Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00138', 'metric_secondary': 'r2=0.9980', 'fallback': 'none'}\n", + "[OK] Right_Arm_Predicted_Mass: {'column': 'Right_Arm_Predicted_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00122', 'metric_secondary': 'r2=0.9980', 'fallback': 'none'}\n", + "[OK] Left_Arm_Fat_Percentage: {'column': 'Left_Arm_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15265', 'metric_secondary': 'r2=0.9986', 'fallback': 'none'}\n", + "[OK] Left_Arm_Fat_Mass: {'column': 'Left_Arm_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00200', 'metric_secondary': 'r2=0.9961', 'fallback': 'none'}\n", + "[OK] Left_Arm_Fat-Free_Mass: {'column': 'Left_Arm_Fat-Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00142', 'metric_secondary': 'r2=0.9980', 'fallback': 'none'}\n", + "[OK] Height_At_10: {'column': 'Height_At_10', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11467', 'metric_secondary': 'r2=0.2967', 'fallback': 'none'}\n", + "[OK] Left_Arm_Predicted_Mass: {'column': 'Left_Arm_Predicted_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00128', 'metric_secondary': 'r2=0.9980', 'fallback': 'none'}\n", + "[OK] Body_Fat_Percentage: {'column': 'Body_Fat_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04620', 'metric_secondary': 'r2=0.9994', 'fallback': 'none'}\n", + "[OK] Reticulocyte_Percentage: {'column': 'Reticulocyte_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07625', 'metric_secondary': 'r2=0.9988', 'fallback': 'none'}\n", + "[OK] Trunk_Fat_Mass: {'column': 'Trunk_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03267', 'metric_secondary': 'r2=0.9988', 'fallback': 'none'}\n", + "[OK] Trunk_Fat-Free_Mass: {'column': 'Trunk_Fat-Free_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01580', 'metric_secondary': 'r2=0.9996', 'fallback': 'none'}\n", + "[OK] Trunk_Predicted_Mass: {'column': 'Trunk_Predicted_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00868', 'metric_secondary': 'r2=0.9997', 'fallback': 'none'}\n", + "[OK] Miserableness: {'column': 'Miserableness', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.14343', 'metric_secondary': 'r2=0.4137', 'fallback': 'none'}\n", + "[OK] Solarium_Use: {'column': 'Solarium_Use', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=18.99546', 'metric_secondary': 'r2=-0.0101', 'fallback': 'median'}\n", + "[OK] Weekly_Walking_Days: {'column': 'Weekly_Walking_Days', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00580', 'metric_secondary': 'r2=0.7195', 'fallback': 'none'}\n", + "[OK] Other_Serious_Medical_Conditions: {'column': 'Other_Serious_Medical_Conditions', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11604', 'metric_secondary': 'r2=0.2884', 'fallback': 'none'}\n", + "[OK] Bread: {'column': 'Bread', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=60.18592', 'metric_secondary': 'r2=0.2012', 'fallback': 'none'}\n", + "[OK] Whole_Body_Fat_Mass: {'column': 'Whole_Body_Fat_Mass', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.06344', 'metric_secondary': 'r2=0.9993', 'fallback': 'none'}\n", + "[OK] Body_Size_At_10: {'column': 'Body_Size_At_10', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19814', 'metric_secondary': 'r2=0.1080', 'fallback': 'none'}\n", + "[OK] Wheezing: {'column': 'Wheezing', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11852', 'metric_secondary': 'r2=0.2935', 'fallback': 'none'}\n", + "[OK] Fed-Up_Feelings: {'column': 'Fed-Up_Feelings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13092', 'metric_secondary': 'r2=0.4578', 'fallback': 'none'}\n", + "[OK] Longstanding_Illness/Disability: {'column': 'Longstanding_Illness/Disability', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.12172', 'metric_secondary': 'r2=0.4460', 'fallback': 'none'}\n", + "[OK] Mood_Swings: {'column': 'Mood_Swings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13213', 'metric_secondary': 'r2=0.4663', 'fallback': 'none'}\n", + "[OK] Nervous_Feelings: {'column': 'Nervous_Feelings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.10714', 'metric_secondary': 'r2=0.4019', 'fallback': 'none'}\n", + "[OK] Worrier/Anxious_Feelings: {'column': 'Worrier/Anxious_Feelings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15546', 'metric_secondary': 'r2=0.3665', 'fallback': 'none'}\n", + "[OK] Guilty_Feelings: {'column': 'Guilty_Feelings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.14841', 'metric_secondary': 'r2=0.2760', 'fallback': 'none'}\n", + "[OK] Sensitivity_to_Hurt_Feelings: {'column': 'Sensitivity_to_Hurt_Feelings', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17205', 'metric_secondary': 'r2=0.3037', 'fallback': 'none'}\n", + "[OK] Tiredness/Lethargy_Frequency: {'column': 'Tiredness/Lethargy_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17287', 'metric_secondary': 'r2=0.3053', 'fallback': 'none'}\n", + "[OK] Enzymatic_In_Urine: {'column': 'Enzymatic_In_Urine', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=12236521.12770', 'metric_secondary': 'r2=0.6405', 'fallback': 'none'}\n", + "[OK] Tanning_Ease: {'column': 'Tanning_Ease', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13398', 'metric_secondary': 'r2=0.0575', 'fallback': 'none'}\n", + "[OK] Able_to_Confide: {'column': 'Able_to_Confide', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11438', 'metric_secondary': 'r2=0.0904', 'fallback': 'none'}\n", + "[OK] Sodium_Urine: {'column': 'Sodium_Urine', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1092.01791', 'metric_secondary': 'r2=0.4494', 'fallback': 'none'}\n", + "[OK] Potassium_Urine: {'column': 'Potassium_Urine', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=462.58147', 'metric_secondary': 'r2=0.5949', 'fallback': 'none'}\n", + "[OK] Unenthusiasm/Disinterest_Frequency: {'column': 'Unenthusiasm/Disinterest_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09237', 'metric_secondary': 'r2=0.4486', 'fallback': 'none'}\n", + "[OK] Tense/Highly_Strung: {'column': 'Tense/Highly_Strung', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09239', 'metric_secondary': 'r2=0.3666', 'fallback': 'none'}\n", + "[OK] Risk-Taking_Behavior: {'column': 'Risk-Taking_Behavior', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16802', 'metric_secondary': 'r2=0.1462', 'fallback': 'none'}\n", + "[OK] Suffering_from_Nerves: {'column': 'Suffering_from_Nerves', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.10314', 'metric_secondary': 'r2=0.3780', 'fallback': 'none'}\n", + "[OK] Tenseness/Restlessness_Frequency: {'column': 'Tenseness/Restlessness_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11827', 'metric_secondary': 'r2=0.3879', 'fallback': 'none'}\n", + "[OK] Post-Embarrassment_Worry: {'column': 'Post-Embarrassment_Worry', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16919', 'metric_secondary': 'r2=0.3221', 'fallback': 'none'}\n", + "[OK] Depressed_Mood_Frequency: {'column': 'Depressed_Mood_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08407', 'metric_secondary': 'r2=0.5371', 'fallback': 'none'}\n", + "[OK] WBC_Count: {'column': 'WBC_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00073', 'metric_secondary': 'r2=0.9958', 'fallback': 'none'}\n", + "[OK] RBC_Count: {'column': 'RBC_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03964', 'metric_secondary': 'r2=0.9969', 'fallback': 'none'}\n", + "[OK] Corpuscular_Haemoglobin_Concentration: {'column': 'Corpuscular_Haemoglobin_Concentration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.51932', 'metric_secondary': 'r2=0.4514', 'fallback': 'none'}\n", + "[OK] Haemoglobin_Concentration: {'column': 'Haemoglobin_Concentration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00387', 'metric_secondary': 'r2=0.9975', 'fallback': 'none'}\n", + "[OK] Mean_Corpuscular_Haemoglobin: {'column': 'Mean_Corpuscular_Haemoglobin', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13281', 'metric_secondary': 'r2=0.9651', 'fallback': 'none'}\n", + "[OK] Haematocrit: {'column': 'Haematocrit', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.27617', 'metric_secondary': 'r2=0.9871', 'fallback': 'none'}\n", + "[OK] Mean_Corpuscular_Haemoglobin_Concentration: {'column': 'Mean_Corpuscular_Haemoglobin_Concentration', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11281', 'metric_secondary': 'r2=0.9101', 'fallback': 'none'}\n", + "[OK] Blood_Cell_Erythrocyte_Distribution_Width: {'column': 'Blood_Cell_Erythrocyte_Distribution_Width', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=31.01486', 'metric_secondary': 'r2=0.9913', 'fallback': 'none'}\n", + "[OK] Platelet_Count: {'column': 'Platelet_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00003', 'metric_secondary': 'r2=0.9858', 'fallback': 'none'}\n", + "[OK] Platelet_Crit: {'column': 'Platelet_Crit', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01272', 'metric_secondary': 'r2=0.9890', 'fallback': 'none'}\n", + "[OK] Platelet_Thrombocyte_Volume: {'column': 'Platelet_Thrombocyte_Volume', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19092', 'metric_secondary': 'r2=0.3061', 'fallback': 'none'}\n", + "[OK] Environment_Score: {'column': 'Environment_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.34719', 'metric_secondary': 'r2=0.9291', 'fallback': 'none'}\n", + "[OK] Irritability: {'column': 'Irritability', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.14319', 'metric_secondary': 'r2=0.2874', 'fallback': 'none'}\n", + "[OK] Hearing_Difficulty: {'column': 'Hearing_Difficulty', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17787', 'metric_secondary': 'r2=0.0649', 'fallback': 'none'}\n", + "[OK] Red_Blood_Cell_(Count): {'column': 'Red_Blood_Cell_(Count)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16398', 'metric_secondary': 'r2=0.9785', 'fallback': 'none'}\n", + "[OK] Eosinophill_Percentage: {'column': 'Eosinophill_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02639', 'metric_secondary': 'r2=0.9219', 'fallback': 'none'}\n", + "[OK] Lymphocyte_Percentage: {'column': 'Lymphocyte_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17163', 'metric_secondary': 'r2=0.9969', 'fallback': 'none'}\n", + "[OK] Neutrophill_Percentage: {'column': 'Neutrophill_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07561', 'metric_secondary': 'r2=0.9781', 'fallback': 'none'}\n", + "[OK] Monocyte_Percentage: {'column': 'Monocyte_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.30818', 'metric_secondary': 'r2=0.9958', 'fallback': 'none'}\n", + "[OK] Eosinophill_Count: {'column': 'Eosinophill_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00057', 'metric_secondary': 'r2=0.9710', 'fallback': 'none'}\n", + "[OK] Platelet_Width: {'column': 'Platelet_Width', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.30995', 'metric_secondary': 'r2=0.7349', 'fallback': 'none'}\n", + "[OK] Neutrophill_Count: {'column': 'Neutrophill_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00027', 'metric_secondary': 'r2=0.8899', 'fallback': 'none'}\n", + "[OK] Lymphocyte_Count: {'column': 'Lymphocyte_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00223', 'metric_secondary': 'r2=0.9527', 'fallback': 'none'}\n", + "[OK] Monocyte_Count: {'column': 'Monocyte_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04233', 'metric_secondary': 'r2=0.9799', 'fallback': 'none'}\n", + "[OK] Basophill_Count: {'column': 'Basophill_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00009', 'metric_secondary': 'r2=0.8739', 'fallback': 'none'}\n", + "[OK] Nucleated_Red_Blood_Cell_Percentage: {'column': 'Nucleated_Red_Blood_Cell_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01141', 'metric_secondary': 'r2=0.9213', 'fallback': 'none'}\n", + "[OK] Weekly_Moderate_Activity_Days: {'column': 'Weekly_Moderate_Activity_Days', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01736', 'metric_secondary': 'r2=0.8440', 'fallback': 'none'}\n", + "[OK] Weekly_Vigorous_Activity_Days: {'column': 'Weekly_Vigorous_Activity_Days', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04184', 'metric_secondary': 'r2=0.8211', 'fallback': 'none'}\n", + "[OK] Diastolic_BP: {'column': 'Diastolic_BP', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=38.04323', 'metric_secondary': 'r2=0.6319', 'fallback': 'none'}\n", + "[OK] Pulse_Rate: {'column': 'Pulse_Rate', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=91.16690', 'metric_secondary': 'r2=0.2794', 'fallback': 'none'}\n", + "[OK] Systolic_BP: {'column': 'Systolic_BP', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=133.03961', 'metric_secondary': 'r2=0.6182', 'fallback': 'none'}\n", + "[OK] Basophill_Percentage: {'column': 'Basophill_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09672', 'metric_secondary': 'r2=0.9015', 'fallback': 'none'}\n", + "[OK] Reticulocyte_Count: {'column': 'Reticulocyte_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00004', 'metric_secondary': 'r2=0.9773', 'fallback': 'none'}\n", + "[OK] High_Light_Scatter_Reticulocyte_Count: {'column': 'High_Light_Scatter_Reticulocyte_Count', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00000', 'metric_secondary': 'r2=0.9821', 'fallback': 'none'}\n", + "[OK] Sphered_Cell_Volume: {'column': 'Sphered_Cell_Volume', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=7.33223', 'metric_secondary': 'r2=0.7418', 'fallback': 'none'}\n", + "[OK] Light_Scatter_Reticulocyte_Percentage: {'column': 'Light_Scatter_Reticulocyte_Percentage', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.12540', 'metric_secondary': 'r2=0.3288', 'fallback': 'none'}\n", + "[OK] Immature_Fraction: {'column': 'Immature_Fraction', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00002', 'metric_secondary': 'r2=0.9941', 'fallback': 'none'}\n", + "[OK] Mean_Reticulocyte_Volume: {'column': 'Mean_Reticulocyte_Volume', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=19.21475', 'metric_secondary': 'r2=0.6895', 'fallback': 'none'}\n", + "[OK] Alkaline_Phosphatase: {'column': 'Alkaline_Phosphatase', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=507.44511', 'metric_secondary': 'r2=0.2811', 'fallback': 'none'}\n", + "[OK] Cholesterol: {'column': 'Cholesterol', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02822', 'metric_secondary': 'r2=0.9782', 'fallback': 'none'}\n", + "[OK] Cystatin_C: {'column': 'Cystatin_C', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01332', 'metric_secondary': 'r2=0.6313', 'fallback': 'none'}\n", + "[OK] Alanine_Aminotransferase: {'column': 'Alanine_Aminotransferase', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=70.46189', 'metric_secondary': 'r2=0.6845', 'fallback': 'none'}\n", + "[OK] Gamma_Glutamyltransferase: {'column': 'Gamma_Glutamyltransferase', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=923.75392', 'metric_secondary': 'r2=0.4866', 'fallback': 'none'}\n", + "[OK] Creatinine_Creatinine: {'column': 'Creatinine_Creatinine', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=112.35011', 'metric_secondary': 'r2=0.6777', 'fallback': 'none'}\n", + "[OK] Urea_Urea: {'column': 'Urea_Urea', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.15129', 'metric_secondary': 'r2=0.4171', 'fallback': 'none'}\n", + "[OK] Triglycerides_Triglycerides: {'column': 'Triglycerides_Triglycerides', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.27655', 'metric_secondary': 'r2=0.7399', 'fallback': 'none'}\n", + "[OK] Urate_Urate: {'column': 'Urate_Urate', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2878.89471', 'metric_secondary': 'r2=0.5535', 'fallback': 'none'}\n", + "[OK] LDL_Cholesterol: {'column': 'LDL_Cholesterol', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01279', 'metric_secondary': 'r2=0.9833', 'fallback': 'none'}\n", + "[OK] C_Protein: {'column': 'C_Protein', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=12.72532', 'metric_secondary': 'r2=0.2974', 'fallback': 'none'}\n", + "[OK] Summer_Outdoors_Time: {'column': 'Summer_Outdoors_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=2.91764', 'metric_secondary': 'r2=0.5099', 'fallback': 'none'}\n", + "[OK] Winter_Outdoors_Time: {'column': 'Winter_Outdoors_Time', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.55315', 'metric_secondary': 'r2=0.5726', 'fallback': 'none'}\n", + "[OK] Aspartate_Aminotransferase: {'column': 'Aspartate_Aminotransferase', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=42.55151', 'metric_secondary': 'r2=0.6339', 'fallback': 'none'}\n", + "[OK] Total_Bilirubin: {'column': 'Total_Bilirubin', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.91629', 'metric_secondary': 'r2=0.9018', 'fallback': 'none'}\n", + "[OK] Apolipoprotein_B: {'column': 'Apolipoprotein_B', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00312', 'metric_secondary': 'r2=0.9448', 'fallback': 'none'}\n", + "[OK] Igf_1: {'column': 'Igf_1', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=24.56654', 'metric_secondary': 'r2=0.2554', 'fallback': 'none'}\n", + "[OK] Haemoglobin_(Hba1C): {'column': 'Haemoglobin_(Hba1C)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=16.14193', 'metric_secondary': 'r2=0.6392', 'fallback': 'none'}\n", + "[OK] Snoring: {'column': 'Snoring', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20626', 'metric_secondary': 'r2=0.1223', 'fallback': 'none'}\n", + "[OK] NOx_Air_2010: {'column': 'NOx_Air_2010', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20355', 'metric_secondary': 'r2=0.9439', 'fallback': 'none'}\n", + "[OK] PM2.5_Absorbance_2010: {'column': 'PM2.5_Absorbance_2010', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01840', 'metric_secondary': 'r2=0.9772', 'fallback': 'none'}\n", + "[OK] PM10_Air_2010: {'column': 'PM10_Air_2010', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02771', 'metric_secondary': 'r2=0.9750', 'fallback': 'none'}\n", + "[OK] PM2.5_Air_2010: {'column': 'PM2.5_Air_2010', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00660', 'metric_secondary': 'r2=0.9099', 'fallback': 'none'}\n", + "[OK] Caffeine_Drink: {'column': 'Caffeine_Drink', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.02196', 'metric_secondary': 'r2=0.0244', 'fallback': 'median'}\n", + "[OK] Inhaler_Use: {'column': 'Inhaler_Use', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.00735', 'metric_secondary': 'r2=0.0481', 'fallback': 'median'}\n", + "[OK] Facial_Ageing: {'column': 'Facial_Ageing', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.18129', 'metric_secondary': 'r2=0.0603', 'fallback': 'none'}\n", + "[OK] FVC: {'column': 'FVC', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08923', 'metric_secondary': 'r2=0.9174', 'fallback': 'none'}\n", + "[OK] PEF: {'column': 'PEF', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4913.93520', 'metric_secondary': 'r2=0.7044', 'fallback': 'none'}\n", + "[OK] FEV1: {'column': 'FEV1', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02576', 'metric_secondary': 'r2=0.9588', 'fallback': 'none'}\n", + "[OK] Over_Speed_Driving: {'column': 'Over_Speed_Driving', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19525', 'metric_secondary': 'r2=0.2133', 'fallback': 'none'}\n", + "[OK] Chronotype: {'column': 'Chronotype', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.22260', 'metric_secondary': 'r2=0.0284', 'fallback': 'median'}\n", + "[OK] Garden_1000m: {'column': 'Garden_1000m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.81392', 'metric_secondary': 'r2=0.8715', 'fallback': 'none'}\n", + "[OK] Water_1000m: {'column': 'Water_1000m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=15.67364', 'metric_secondary': 'r2=0.9709', 'fallback': 'none'}\n", + "[OK] Garden_300m: {'column': 'Garden_300m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1.14739', 'metric_secondary': 'r2=0.8690', 'fallback': 'none'}\n", + "[OK] Noise_Level: {'column': 'Noise_Level', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4.63175', 'metric_secondary': 'r2=0.9901', 'fallback': 'none'}\n", + "[OK] Greenspace_300m: {'column': 'Greenspace_300m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=12.93868', 'metric_secondary': 'r2=0.9407', 'fallback': 'none'}\n", + "[OK] Greenspace_1000m: {'column': 'Greenspace_1000m', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=5.34329', 'metric_secondary': 'r2=0.9586', 'fallback': 'none'}\n", + "[OK] First_Sexual_Intercourse: {'column': 'First_Sexual_Intercourse', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=11.86485', 'metric_secondary': 'r2=0.1890', 'fallback': 'none'}\n", + "[OK] Crime_Score: {'column': 'Crime_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=26.45542', 'metric_secondary': 'r2=0.8875', 'fallback': 'none'}\n", + "[OK] Education_Score: {'column': 'Education_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=14.24937', 'metric_secondary': 'r2=0.8606', 'fallback': 'none'}\n", + "[OK] Housing_Score: {'column': 'Housing_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.07317', 'metric_secondary': 'r2=0.8805', 'fallback': 'none'}\n", + "[OK] Income_Score: {'column': 'Income_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00006', 'metric_secondary': 'r2=0.9832', 'fallback': 'none'}\n", + "[OK] Coast_Distance: {'column': 'Coast_Distance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00019', 'metric_secondary': 'r2=0.9801', 'fallback': 'none'}\n", + "[OK] Index_of_Multiple_Deprivation: {'column': 'Index_of_Multiple_Deprivation', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.52451', 'metric_secondary': 'r2=0.9973', 'fallback': 'none'}\n", + "[OK] Employment_Score: {'column': 'Employment_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=15.87962', 'metric_secondary': 'r2=0.9384', 'fallback': 'none'}\n", + "[OK] Albumin_Albumin: {'column': 'Albumin_Albumin', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=3.54718', 'metric_secondary': 'r2=0.4850', 'fallback': 'none'}\n", + "[OK] Calcium_Calcium: {'column': 'Calcium_Calcium', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00511', 'metric_secondary': 'r2=0.4311', 'fallback': 'none'}\n", + "[OK] HDL_Cholesterol: {'column': 'HDL_Cholesterol', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00749', 'metric_secondary': 'r2=0.9493', 'fallback': 'none'}\n", + "[OK] Total_Protein: {'column': 'Total_Protein', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=9.22529', 'metric_secondary': 'r2=0.4502', 'fallback': 'none'}\n", + "[OK] Glucose_Glucose: {'column': 'Glucose_Glucose', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.66192', 'metric_secondary': 'r2=0.6126', 'fallback': 'none'}\n", + "[OK] Phosphate_Phosphate: {'column': 'Phosphate_Phosphate', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01922', 'metric_secondary': 'r2=0.2563', 'fallback': 'none'}\n", + "[OK] Apolipoprotein_A: {'column': 'Apolipoprotein_A', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00739', 'metric_secondary': 'r2=0.8998', 'fallback': 'none'}\n", + "[OK] Shbg: {'column': 'Shbg', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=360.79332', 'metric_secondary': 'r2=0.5249', 'fallback': 'none'}\n", + "[OK] Testosterone: {'column': 'Testosterone', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4.26155', 'metric_secondary': 'r2=0.8839', 'fallback': 'none'}\n", + "[OK] Direct_Bilirubin: {'column': 'Direct_Bilirubin', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.05971', 'metric_secondary': 'r2=0.9132', 'fallback': 'none'}\n", + "[OK] MET_Moderate: {'column': 'MET_Moderate', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=5086.36089', 'metric_secondary': 'r2=0.9965', 'fallback': 'none'}\n", + "[OK] Activity_Days: {'column': 'Activity_Days', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.29166', 'metric_secondary': 'r2=0.9875', 'fallback': 'none'}\n", + "[OK] Activity_Recommendation: {'column': 'Activity_Recommendation', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00002', 'metric_secondary': 'r2=0.9999', 'fallback': 'none'}\n", + "[OK] MET_Walking: {'column': 'MET_Walking', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=5936.64890', 'metric_secondary': 'r2=0.9948', 'fallback': 'none'}\n", + "[OK] MET_Vigorous: {'column': 'MET_Vigorous', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=6285.74163', 'metric_secondary': 'r2=0.9954', 'fallback': 'none'}\n", + "[OK] IPAQ_Activity_Group: {'column': 'IPAQ_Activity_Group', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00048', 'metric_secondary': 'r2=0.9968', 'fallback': 'none'}\n", + "[OK] Activity_Minutes: {'column': 'Activity_Minutes', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=104.87300', 'metric_secondary': 'r2=0.9896', 'fallback': 'none'}\n", + "[OK] Walking_Recommendation_Compliance: {'column': 'Walking_Recommendation_Compliance', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00087', 'metric_secondary': 'r2=0.9941', 'fallback': 'none'}\n", + "[OK] Total_MET_Minutes_per_Week: {'column': 'Total_MET_Minutes_per_Week', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=6335.64322', 'metric_secondary': 'r2=0.9991', 'fallback': 'none'}\n", + "[OK] Breastfed_Baby: {'column': 'Breastfed_Baby', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.18390', 'metric_secondary': 'r2=0.0789', 'fallback': 'none'}\n", + "[OK] FEV1_Z_Score: {'column': 'FEV1_Z_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00375', 'metric_secondary': 'r2=0.9970', 'fallback': 'none'}\n", + "[OK] FEV1_FVC_Ratio: {'column': 'FEV1_FVC_Ratio', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00927', 'metric_secondary': 'r2=0.9884', 'fallback': 'none'}\n", + "[OK] FVC_Z_Score: {'column': 'FVC_Z_Score', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00535', 'metric_secondary': 'r2=0.9952', 'fallback': 'none'}\n", + "[OK] Spirometry_Quality: {'column': 'Spirometry_Quality', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08186', 'metric_secondary': 'r2=0.0990', 'fallback': 'none'}\n", + "[OK] Sunburns: {'column': 'Sunburns', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.21410', 'metric_secondary': 'r2=0.1374', 'fallback': 'none'}\n", + "[OK] Lipoprotein_A: {'column': 'Lipoprotein_A', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=2356.15944', 'metric_secondary': 'r2=0.0215', 'fallback': 'median'}\n", + "[OK] FEV1_(Best_Measure): {'column': 'FEV1_(Best_Measure)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.00642', 'metric_secondary': 'r2=0.9895', 'fallback': 'none'}\n", + "[OK] FVC_(Best_Measure): {'column': 'FVC_(Best_Measure)', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.01083', 'metric_secondary': 'r2=0.9889', 'fallback': 'none'}\n", + "[OK] Email_Access: {'column': 'Email_Access', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.06467', 'metric_secondary': 'r2=0.0258', 'fallback': 'median'}\n", + "[OK] Bowel_Open_Min: {'column': 'Bowel_Open_Min', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=7295.04911', 'metric_secondary': 'r2=0.2187', 'fallback': 'none'}\n", + "[OK] Bowel_Open_Max: {'column': 'Bowel_Open_Max', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=1907.53442', 'metric_secondary': 'r2=0.4941', 'fallback': 'none'}\n", + "[OK] Bowel_Open_Average: {'column': 'Bowel_Open_Average', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=4068.44912', 'metric_secondary': 'r2=0.3667', 'fallback': 'none'}\n", + "[OK] Headache: {'column': 'Headache', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.18840', 'metric_secondary': 'r2=0.2088', 'fallback': 'none'}\n", + "[OK] Breath_Shortness: {'column': 'Breath_Shortness', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16005', 'metric_secondary': 'r2=0.2642', 'fallback': 'none'}\n", + "[OK] Heart_Pounding: {'column': 'Heart_Pounding', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.17364', 'metric_secondary': 'r2=0.1805', 'fallback': 'none'}\n", + "[OK] Back_Pain: {'column': 'Back_Pain', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20411', 'metric_secondary': 'r2=0.1495', 'fallback': 'none'}\n", + "[OK] Limb_Joint_Pain: {'column': 'Limb_Joint_Pain', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16754', 'metric_secondary': 'r2=0.1635', 'fallback': 'none'}\n", + "[OK] Tiredness: {'column': 'Tiredness', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15552', 'metric_secondary': 'r2=0.3162', 'fallback': 'none'}\n", + "[OK] Sleep_Trouble: {'column': 'Sleep_Trouble', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16397', 'metric_secondary': 'r2=0.2664', 'fallback': 'none'}\n", + "[OK] Dizziness: {'column': 'Dizziness', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.16016', 'metric_secondary': 'r2=0.1864', 'fallback': 'none'}\n", + "[OK] Nausea: {'column': 'Nausea', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.10358', 'metric_secondary': 'r2=0.1875', 'fallback': 'none'}\n", + "[OK] Chest_Pain: {'column': 'Chest_Pain', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09515', 'metric_secondary': 'r2=0.2064', 'fallback': 'none'}\n", + "[OK] Fainting: {'column': 'Fainting', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.02884', 'metric_secondary': 'r2=0.0934', 'fallback': 'none'}\n", + "[OK] Loose_Stools_Frequency: {'column': 'Loose_Stools_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.19391', 'metric_secondary': 'r2=0.1692', 'fallback': 'none'}\n", + "[OK] Hard_Stools_Frequency: {'column': 'Hard_Stools_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.20094', 'metric_secondary': 'r2=0.1928', 'fallback': 'none'}\n", + "[OK] Urinary_Frequency: {'column': 'Urinary_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.22031', 'metric_secondary': 'r2=0.1083', 'fallback': 'none'}\n", + "[OK] Abdomen_Pain_Frequency: {'column': 'Abdomen_Pain_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.13654', 'metric_secondary': 'r2=0.4499', 'fallback': 'none'}\n", + "[OK] Recent_Abdominal_Pain: {'column': 'Recent_Abdominal_Pain', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11318', 'metric_secondary': 'r2=0.4575', 'fallback': 'none'}\n", + "[OK] Abdominal_Distension: {'column': 'Abdominal_Distension', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.12470', 'metric_secondary': 'r2=0.3120', 'fallback': 'none'}\n", + "[OK] Bowel_Satisfaction: {'column': 'Bowel_Satisfaction', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11267', 'metric_secondary': 'r2=0.2667', 'fallback': 'none'}\n", + "[OK] Bowel_Interference: {'column': 'Bowel_Interference', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.15387', 'metric_secondary': 'r2=0.3840', 'fallback': 'none'}\n", + "[OK] Coeliac_Disease: {'column': 'Coeliac_Disease', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.01618', 'metric_secondary': 'r2=0.0477', 'fallback': 'median'}\n", + "[OK] IBS: {'column': 'IBS', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08246', 'metric_secondary': 'r2=0.2479', 'fallback': 'none'}\n", + "[OK] Caesarian_Born: {'column': 'Caesarian_Born', 'type': 'regression', 'trained': False, 'metric_primary': 'mse=0.02584', 'metric_secondary': 'r2=0.0132', 'fallback': 'median'}\n", + "[OK] Sensitive_Stomach: {'column': 'Sensitive_Stomach', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.12573', 'metric_secondary': 'r2=0.2929', 'fallback': 'none'}\n", + "[OK] Childhood_Antibiotics: {'column': 'Childhood_Antibiotics', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.11160', 'metric_secondary': 'r2=0.0798', 'fallback': 'none'}\n", + "[OK] Alcohol-Related_Injury: {'column': 'Alcohol-Related_Injury', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03446', 'metric_secondary': 'r2=0.0933', 'fallback': 'none'}\n", + "[OK] Alcohol_Drinking_Frequency: {'column': 'Alcohol_Drinking_Frequency', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.04356', 'metric_secondary': 'r2=0.4509', 'fallback': 'none'}\n", + "[OK] Serious_Accident: {'column': 'Serious_Accident', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.08273', 'metric_secondary': 'r2=0.0628', 'fallback': 'none'}\n", + "[OK] Combat_War_Exposure: {'column': 'Combat_War_Exposure', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.03397', 'metric_secondary': 'r2=0.0647', 'fallback': 'none'}\n", + "[OK] Appetite_Changes: {'column': 'Appetite_Changes', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.10221', 'metric_secondary': 'r2=0.3431', 'fallback': 'none'}\n", + "[OK] Stressful_Thoughts: {'column': 'Stressful_Thoughts', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.12213', 'metric_secondary': 'r2=0.3859', 'fallback': 'none'}\n", + "[OK] Concentration_Issues: {'column': 'Concentration_Issues', 'type': 'regression', 'trained': True, 'metric_primary': 'mse=0.09228', 'metric_secondary': 'r2=0.3689', 'fallback': 'none'}\n" + ] + } + ], + "source": [ + "# =======================\n", + "# Notebook 一体化：XGBoost 逐列机器学习插补（兼容旧版 xgboost，无 early_stopping_rounds）\n", + "# =======================\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import LabelEncoder\n", + "from sklearn.metrics import accuracy_score, f1_score, mean_squared_error, r2_score\n", + "\n", + "# xgboost 基本导入\n", + "try:\n", + " import xgboost as xgb\n", + " from xgboost import XGBClassifier, XGBRegressor\n", + "except Exception as e:\n", + " raise RuntimeError(\"需要已安装 xgboost。请先在该环境安装：pip install xgboost\") from e\n", + "\n", + "# ============ Config（按需修改） ============\n", + "INPUT_CSV = \"/content/drive/MyDrive/demo_200000.csv\"\n", + "OUTPUT_CSV = \"demo_200000_imputed.csv\"\n", + "REPORT_CSV = \"demo_200000_impute_report.csv\"\n", + "\n", + "# 不参与作为特征/目标的列（如ID/标签）\n", + "EXCLUDE_COLS = [\"ID\", \"DR\",\"DR_time\",\"AMD_time\",\"AMD\",\"glaucoma_time\",\"glaucoma\",\"cataract_time\",\"cataract\"]\n", + "\n", + "# 类别压帽（最多保留前N个高频类别，其余合并为 OTHER）\n", + "MAX_CATEGORIES = 50\n", + "\n", + "# 训练与评估\n", + "TEST_SIZE = 0.2\n", + "RANDOM_STATE = 42\n", + "N_ESTIMATORS = 300 # 如果耗时长，可先降到 200\n", + "LEARNING_RATE = 0.05\n", + "MAX_DEPTH = 6\n", + "SUBSAMPLE = 0.8\n", + "COLSAMPLE_BYTREE = 0.8\n", + "\n", + "# 早停轮数（老版本不支持 fit 参数，我们用回调；若回调也不可用就自动跳过）\n", + "EARLY_STOPPING_ROUNDS = 50\n", + "\n", + "# 分类任务：最低可接受准确率（低于则回退到众数）\n", + "EVAL_ACC_THRESHOLD = 0.65\n", + "# 回归任务：模型MSE需 <= baseline*MSE_RATIO 才接受（即至少优于均值/中位数约5%）\n", + "EVAL_MSE_RATIO = 0.95\n", + "\n", + "# XGBoost tree_method（可切到 \"gpu_hist\" 如果你的环境支持 GPU）\n", + "XGB_TREE_METHOD = \"hist\" # \"hist\" | \"approx\" | \"auto\" | \"gpu_hist\"\n", + "\n", + "\n", + "# ============ Helpers ============\n", + "def is_numeric_series(s: pd.Series) -> bool:\n", + " return pd.api.types.is_integer_dtype(s) or pd.api.types.is_float_dtype(s)\n", + "\n", + "def cap_categories(series: pd.Series, max_categories: int = 50):\n", + " \"\"\"保留前N高频类别，其余 -> 'OTHER'\"\"\"\n", + " vc = series.value_counts(dropna=False)\n", + " top = set(vc.head(max_categories).index.tolist())\n", + " return series.apply(lambda x: x if x in top else \"OTHER\")\n", + "\n", + "def one_hot_fit_transform(df: pd.DataFrame, categorical_cols, max_categories: int):\n", + " \"\"\"拟合并独热编码，返回：编码后DF、meta（每列类别集合）、最终列名列表\"\"\"\n", + " df = df.copy()\n", + " meta = {}\n", + " for col in categorical_cols:\n", + " s = df[col].astype(str).fillna(\"UNKNOWN\")\n", + " s = cap_categories(s, max_categories=max_categories)\n", + " df[col] = s\n", + " meta[col] = sorted(df[col].unique().tolist())\n", + " dummied = pd.get_dummies(df, columns=categorical_cols, dummy_na=False)\n", + " return dummied, meta, dummied.columns.tolist()\n", + "\n", + "def one_hot_transform_with_meta(df: pd.DataFrame, categorical_cols, meta, all_cols):\n", + " \"\"\"用拟合阶段的 meta 做独热，并对齐列\"\"\"\n", + " df = df.copy()\n", + " for col in categorical_cols:\n", + " s = df[col].astype(str).fillna(\"UNKNOWN\")\n", + " df[col] = s.apply(lambda x: x if x in meta[col] else \"OTHER\")\n", + " dummied = pd.get_dummies(df, columns=categorical_cols, dummy_na=False)\n", + " for c in all_cols:\n", + " if c not in dummied.columns:\n", + " dummied[c] = 0\n", + " dummied = dummied[all_cols]\n", + " return dummied\n", + "\n", + "def evaluate_classifier(y_true, y_pred):\n", + " acc = accuracy_score(y_true, y_pred)\n", + " f1m = f1_score(y_true, y_pred, average=\"macro\")\n", + " return {\"accuracy\": acc, \"f1_macro\": f1m}\n", + "\n", + "def evaluate_regressor(y_true, y_pred):\n", + " mse = mean_squared_error(y_true, y_pred)\n", + " r2 = r2_score(y_true, y_pred)\n", + " return {\"mse\": mse, \"r2\": r2}\n", + "\n", + "def _fit_with_optional_early_stopping(model, X_tr, y_tr, X_va, y_va):\n", + " \"\"\"\n", + " 兼容不同 xgboost 版本的早停：\n", + " - 优先使用 xgboost.callback.EarlyStopping\n", + " - 如果不可用，直接不做早停\n", + " \"\"\"\n", + " callbacks = []\n", + " eval_set = [(X_va, y_va)]\n", + "\n", + " # 优先用官方回调（老版本也支持）\n", + " try:\n", + " cb = xgb.callback.EarlyStopping(\n", + " rounds=EARLY_STOPPING_ROUNDS,\n", + " save_best=True,\n", + " maximize=False # 回归/分类默认都是最小化损失\n", + " )\n", + " callbacks.append(cb)\n", + " model.fit(X_tr, y_tr, eval_set=eval_set, callbacks=callbacks, verbose=False)\n", + " return model\n", + " except Exception:\n", + " # 无法使用回调则直接无早停训练\n", + " model.fit(X_tr, y_tr, eval_set=eval_set, verbose=False)\n", + " return model\n", + "\n", + "def xgb_impute_column(\n", + " df: pd.DataFrame,\n", + " target_col: str,\n", + " exclude_cols: list,\n", + " max_categories: int = 50,\n", + " test_size: float = 0.2,\n", + " random_state: int = 42,\n", + " n_estimators: int = 300,\n", + " learning_rate: float = 0.05,\n", + " max_depth: int = 6,\n", + " subsample: float = 0.8,\n", + " colsample_bytree: float = 0.8,\n", + " eval_acc_threshold: float = 0.65,\n", + " eval_mse_ratio: float = 0.95,\n", + " tree_method: str = \"hist\",\n", + "):\n", + " \"\"\"对单列进行 XGB 插补：返回填补后的 Series 与一条报告 dict\"\"\"\n", + " y = df[target_col]\n", + " notnull_mask = y.notna()\n", + " null_mask = ~notnull_mask\n", + " if null_mask.sum() == 0:\n", + " return y, {\"column\": target_col, \"type\": \"skip_no_missing\", \"trained\": False,\n", + " \"metric_primary\": None, \"metric_secondary\": None, \"fallback\": \"none\"}\n", + "\n", + " feature_cols = [c for c in df.columns if c != target_col and c not in exclude_cols]\n", + " X = df[feature_cols].copy()\n", + "\n", + " # 特征侧预填（不改原 df）\n", + " num_cols = [c for c in feature_cols if is_numeric_series(X[c])]\n", + " cat_cols = [c for c in feature_cols if not is_numeric_series(X[c])]\n", + " for c in num_cols:\n", + " X[c] = pd.to_numeric(X[c], errors=\"coerce\").fillna(X[c].median())\n", + " for c in cat_cols:\n", + " X[c] = X[c].astype(str).fillna(\"UNKNOWN\")\n", + "\n", + " X_train_full = X.loc[notnull_mask].copy()\n", + " y_train_full = y.loc[notnull_mask].copy()\n", + " X_null = X.loc[null_mask].copy()\n", + "\n", + " X_train_oh, meta, all_cols = one_hot_fit_transform(X_train_full, cat_cols, max_categories=max_categories)\n", + " X_null_oh = one_hot_transform_with_meta(X_null, cat_cols, meta, all_cols)\n", + "\n", + " # 回归还是分类由目标列类型决定\n", + " if is_numeric_series(y_train_full):\n", + " task = \"regression\"\n", + " y_vec = pd.to_numeric(y_train_full, errors=\"coerce\")\n", + " valid = y_vec.notna()\n", + " X_train_oh = X_train_oh.loc[valid]; y_vec = y_vec.loc[valid]\n", + "\n", + " X_tr, X_te, y_tr, y_te = train_test_split(X_train_oh, y_vec, test_size=test_size, random_state=random_state)\n", + " model = XGBRegressor(\n", + " n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth,\n", + " subsample=subsample, colsample_bytree=colsample_bytree, objective=\"reg:squarederror\",\n", + " tree_method=tree_method, random_state=random_state, n_jobs=-1\n", + " )\n", + " model = _fit_with_optional_early_stopping(model, X_tr, y_tr, X_te, y_te)\n", + " y_pred = model.predict(X_te)\n", + " m = evaluate_regressor(y_te, y_pred)\n", + "\n", + " # 基线：均值/中位数\n", + " mse_model = m[\"mse\"]\n", + " mse_mean = mean_squared_error(y_te, np.full_like(y_te, y_tr.mean(), dtype=float))\n", + " mse_median= mean_squared_error(y_te, np.full_like(y_te, float(np.median(y_tr)), dtype=float))\n", + " best_bl = min(mse_mean, mse_median)\n", + "\n", + " if mse_model <= eval_mse_ratio * best_bl:\n", + " y_null_pred = model.predict(X_null_oh)\n", + " filled = y.copy(); filled.loc[null_mask] = y_null_pred\n", + " rep = {\"column\": target_col, \"type\": task, \"trained\": True,\n", + " \"metric_primary\": f\"mse={m['mse']:.5f}\", \"metric_secondary\": f\"r2={m['r2']:.4f}\",\n", + " \"fallback\": \"none\"}\n", + " return filled, rep\n", + " else:\n", + " filled = y.fillna(y.median())\n", + " rep = {\"column\": target_col, \"type\": task, \"trained\": False,\n", + " \"metric_primary\": f\"mse={m['mse']:.5f}\", \"metric_secondary\": f\"r2={m['r2']:.4f}\",\n", + " \"fallback\": \"median\"}\n", + " return filled, rep\n", + "\n", + " else:\n", + " task = \"classification\"\n", + " y_str = y_train_full.astype(str)\n", + " enc = LabelEncoder(); y_enc = enc.fit_transform(y_str)\n", + "\n", + " X_tr, X_te, y_tr, y_te = train_test_split(\n", + " X_train_oh, y_enc, test_size=test_size, random_state=random_state, stratify=y_enc\n", + " )\n", + " clf = XGBClassifier(\n", + " n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth,\n", + " subsample=subsample, colsample_bytree=colsample_bytree,\n", + " objective=\"multi:softprob\" if len(np.unique(y_enc))>2 else \"binary:logistic\",\n", + " num_class=len(np.unique(y_enc)) if len(np.unique(y_enc))>2 else None,\n", + " tree_method=tree_method, random_state=random_state, n_jobs=-1, use_label_encoder=False\n", + " )\n", + " clf = _fit_with_optional_early_stopping(clf, X_tr, y_tr, X_te, y_te)\n", + " y_pred = clf.predict(X_te)\n", + " m = evaluate_classifier(y_te, y_pred)\n", + "\n", + " if m[\"accuracy\"] >= eval_acc_threshold:\n", + " y_null_pred_enc = clf.predict(X_null_oh)\n", + " y_null_pred = enc.inverse_transform(y_null_pred_enc)\n", + " filled = y.copy(); filled.loc[null_mask] = y_null_pred\n", + " rep = {\"column\": target_col, \"type\": task, \"trained\": True,\n", + " \"metric_primary\": f\"acc={m['accuracy']:.4f}\", \"metric_secondary\": f\"f1_macro={m['f1_macro']:.4f}\",\n", + " \"fallback\": \"none\"}\n", + " return filled, rep\n", + " else:\n", + " mode_val = y_str.mode().iloc[0] if y_str.mode().shape[0] else \"UNKNOWN\"\n", + " filled = y.fillna(mode_val)\n", + " rep = {\"column\": target_col, \"type\": task, \"trained\": False,\n", + " \"metric_primary\": f\"acc={m['accuracy']:.4f}\", \"metric_secondary\": f\"f1_macro={m['f1_macro']:.4f}\",\n", + " \"fallback\": f\"mode({mode_val})\"}\n", + " return filled, rep\n", + "\n", + "\n", + "def impute_dataframe(df: pd.DataFrame, exclude_cols=None, max_categories=50):\n", + " dfw = df.copy()\n", + " exclude_cols = exclude_cols or []\n", + "\n", + " miss_counts = dfw.isna().sum()\n", + " miss_cols = miss_counts[miss_counts > 0].sort_values(ascending=True).index.tolist()\n", + "\n", + " reports = []\n", + " for col in miss_cols:\n", + " if col in exclude_cols:\n", + " reports.append({\"column\": col, \"type\": \"skipped_excluded\", \"trained\": False,\n", + " \"metric_primary\": None, \"metric_secondary\": None, \"fallback\": \"none\"})\n", + " continue\n", + "\n", + " filled_col, rep = xgb_impute_column(\n", + " df=dfw, target_col=col, exclude_cols=exclude_cols,\n", + " max_categories=max_categories, test_size=TEST_SIZE, random_state=RANDOM_STATE,\n", + " n_estimators=N_ESTIMATORS, learning_rate=LEARNING_RATE, max_depth=MAX_DEPTH,\n", + " subsample=SUBSAMPLE, colsample_bytree=COLSAMPLE_BYTREE,\n", + " eval_acc_threshold=EVAL_ACC_THRESHOLD, eval_mse_ratio=EVAL_MSE_RATIO,\n", + " tree_method=XGB_TREE_METHOD\n", + " )\n", + " dfw[col] = filled_col\n", + " reports.append(rep)\n", + " print(f\"[OK] {col}: {rep}\")\n", + "\n", + " report_df = pd.DataFrame(reports, columns=[\"column\",\"type\",\"trained\",\"metric_primary\",\"metric_secondary\",\"fallback\"])\n", + " return dfw, report_df\n", + "\n", + "\n", + "# ============ RUN ============\n", + "df_in = pd.read_csv(INPUT_CSV)\n", + "imputed_df, report_df = impute_dataframe(df_in, exclude_cols=EXCLUDE_COLS, max_categories=MAX_CATEGORIES)\n", + "\n", + "imputed_df.to_csv(OUTPUT_CSV, index=False)\n", + "report_df.to_csv(REPORT_CSV, index=False)\n", + "\n", + "print(\"插补完成。\")\n", + "print(\"Imputed CSV ->\", OUTPUT_CSV)\n", + "print(\"Report CSV ->\", REPORT_CSV)\n", + "\n", + "# 预览\n", + "imputed_df.head(), report_df.head(20)\n" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/jointContribution/AI_Climate_Diseases/paddle_test_example.ipynb b/jointContribution/AI_Climate_Diseases/paddle_test_example.ipynb new file mode 100644 index 0000000000..1543d69f30 --- /dev/null +++ b/jointContribution/AI_Climate_Diseases/paddle_test_example.ipynb @@ -0,0 +1,5735 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "A100" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "code", + "source": [ + "!pip -q install paddlepaddle -i https://pypi.tuna.tsinghua.edu.cn/simple\n", + "\n", + "# 验证\n", + "import paddle\n", + "paddle.utils.run_check()\n", + "print(\"Paddle version:\", paddle.__version__)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "awQrZ0Q-NF7S", + "outputId": "c8c84f16-eccb-4a9d-af47-60a1188792ad" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m189.0/189.0 MB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.5/65.5 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md\n", + " warnings.warn(warning_message)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Running verify PaddlePaddle program ... \n", + "PaddlePaddle works well on 1 CPU.\n", + "PaddlePaddle is installed successfully! Let's start deep learning with PaddlePaddle now.\n", + "Paddle version: 3.2.0\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/paddle/pir/math_op_patch.py:219: UserWarning: Value do not have 'place' interface for pir graph mode, try not to use it. None will be returned.\n", + " warnings.warn(\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import math\n", + "from typing import Literal, Optional\n", + "\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "\n", + "# ---------- 通用初始化 ------------------------------------------------------\n", + "def init_rsqrt_uniform_(w: paddle.Tensor) -> paddle.Tensor:\n", + " bound = 1.0 / math.sqrt(w.shape[-1])\n", + " noise = paddle.uniform(w.shape, min=-bound, max=bound, dtype=w.dtype)\n", + " w.set_value(noise)\n", + " return w\n", + "\n", + "def init_random_signs_(w: paddle.Tensor) -> paddle.Tensor:\n", + " # 0/1 伯努利 -> *2 -1 => {-1, +1}\n", + " with paddle.no_grad():\n", + " p = paddle.full(w.shape, 0.5, dtype='float32')\n", + " s = paddle.bernoulli(p) * 2.0 - 1.0\n", + " s = paddle.cast(s, w.dtype)\n", + " w.set_value(s)\n", + " return w\n", + "\n", + "# ---------- 基础层 ----------------------------------------------------------\n", + "class NLinear(nn.Layer):\n", + " \"\"\"PackedEnsemble: K 份 Linear 打包 → 输入 (B,K,D), 权重布局 (K, I, O)\"\"\"\n", + " def __init__(self, k: int, in_f: int, out_f: int, bias: bool = True):\n", + " super().__init__()\n", + " self.k = k\n", + " self.in_f = in_f\n", + " self.out_f = out_f\n", + " # 按 Paddle 线性层布局 [I, O]\n", + " self.weight = self.create_parameter(shape=[k, in_f, out_f])\n", + " self.bias_e = self.create_parameter(shape=[k, out_f]) if bias else None\n", + " self.reset_parameters()\n", + "\n", + " def reset_parameters(self):\n", + " init_rsqrt_uniform_(self.weight)\n", + " if self.bias_e is not None:\n", + " init_rsqrt_uniform_(self.bias_e)\n", + "\n", + " def forward(self, x): # x: (B,K,D=I)\n", + " # 转成 (K,B,D) 与 batched matmul 对齐\n", + " xk = paddle.transpose(x, [1, 0, 2]) # (K,B,I)\n", + " # (K,B,I) @ (K,I,O) = (K,B,O)\n", + " yk = paddle.bmm(xk, self.weight) # (K,B,O)\n", + " y = paddle.transpose(yk, [1, 0, 2]) # (B,K,O)\n", + " if self.bias_e is not None:\n", + " y = y + self.bias_e # 广播到 (B,K,O)\n", + " return y\n", + "\n", + "class ScaleEnsemble(nn.Layer):\n", + " \"\"\"Mini-Ensemble：每层一个 rank-1 缩放向量\"\"\"\n", + " def __init__(self, k: int, d: int, init='ones'):\n", + " super().__init__()\n", + " self.k = k\n", + " self.d = d\n", + " self.init = init\n", + " self.weight = self.create_parameter(shape=[k, d])\n", + " self.reset_parameters()\n", + "\n", + " def reset_parameters(self):\n", + " if self.init == 'ones':\n", + " self.weight.set_value(paddle.ones_like(self.weight))\n", + " else:\n", + " init_random_signs_(self.weight)\n", + "\n", + " def forward(self, x): # (B,K,D)\n", + " return x * self.weight # 广播到 (B,K,D)\n", + "\n", + "class LinearBE(nn.Layer):\n", + " \"\"\"\n", + " BatchEnsemble Linear（Paddle 布局）：\n", + " 权重 W: [I, O]；前向 y_e = ((x * r_e) @ W) * s_e + b_e\n", + " 输入: x (B,K,I)\n", + " 输出: y (B,K,O)\n", + " \"\"\"\n", + " def __init__(self, in_f: int, out_f: int, k: int,\n", + " scale_init='ones', bias: bool = True):\n", + " super().__init__()\n", + " self.k = k\n", + " self.in_f = in_f\n", + " self.out_f = out_f\n", + " # 显式属性名，避免冲突；按 Paddle 线性层布局 [I, O]\n", + " self.weight = self.create_parameter(shape=[in_f, out_f])\n", + " self.r = self.create_parameter(shape=[k, in_f])\n", + " self.s = self.create_parameter(shape=[k, out_f])\n", + " self.use_bias = bias\n", + " self.bias_e = self.create_parameter(shape=[k, out_f]) if bias else None\n", + " self.scale_init = scale_init\n", + " self.reset_parameters()\n", + "\n", + " def reset_parameters(self):\n", + " init_rsqrt_uniform_(self.weight)\n", + " if self.scale_init == 'ones':\n", + " self.r.set_value(paddle.ones_like(self.r))\n", + " self.s.set_value(paddle.ones_like(self.s))\n", + " else:\n", + " init_random_signs_(self.r)\n", + " init_random_signs_(self.s)\n", + " if self.use_bias:\n", + " init_rsqrt_uniform_(self.bias_e)\n", + "\n", + " def forward(self, x): # (B,K,I)\n", + " xr = x * self.r # (B,K,I)\n", + " # (B,K,I) @ (I,O) = (B,K,O)\n", + " y = paddle.matmul(xr, self.weight) # (B,K,O)\n", + " y = y * self.s # (B,K,O)\n", + " if self.use_bias:\n", + " y = y + self.bias_e\n", + " return y\n", + "\n", + "# ---------- Backbone MLP -----------------------------------------------------\n", + "class MLPBlock(nn.Layer):\n", + " def __init__(self, d_in, d_hid, dropout, act='ReLU'):\n", + " super().__init__()\n", + " Act = getattr(nn, act)\n", + " self.net = nn.Sequential(\n", + " nn.Linear(d_in, d_hid), # Paddle: weight [d_in, d_hid]\n", + " Act(),\n", + " nn.Dropout(dropout),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " # 允许 (B,K,D) 或 (B,D)；Linear 会在最后一维上工作\n", + " return self.net(x)\n", + "\n", + "class BackboneMLP(nn.Layer):\n", + " def __init__(self, n_blocks: int, d_in: int, d_hidden: int, dropout: float):\n", + " super().__init__()\n", + " blocks = []\n", + " for i in range(n_blocks):\n", + " blocks.append(\n", + " MLPBlock(d_in if i == 0 else d_hidden, d_hidden, dropout)\n", + " )\n", + " self.blocks = nn.LayerList(blocks)\n", + "\n", + " def forward(self, x):\n", + " for blk in self.blocks:\n", + " x = blk(x)\n", + " return x\n", + "\n", + "# ---------- 工具：递归替换 Linear 为 BE / Packed ---------------------------\n", + "def _get_parent_by_path(root: nn.Layer, path_list):\n", + " \"\"\"根据命名路径拿到父层（最后一个名是子层名）\"\"\"\n", + " cur = root\n", + " for p in path_list:\n", + " if hasattr(cur, p):\n", + " cur = getattr(cur, p)\n", + " else:\n", + " sub_layers = getattr(cur, \"_sub_layers\", None)\n", + " if sub_layers is None or p not in sub_layers:\n", + " raise AttributeError(f\"Cannot locate sublayer '{p}' under '{type(cur).__name__}'\")\n", + " cur = sub_layers[p]\n", + " return cur\n", + "\n", + "def _replace_linear(module: nn.Layer, k: int, mode: Literal['be', 'packed']):\n", + " \"\"\"\n", + " 遍历 module 的子层，把 nn.Linear 替换为 LinearBE 或 NLinear\n", + " 注意：Paddle Linear 的 weight 形状为 [in_features, out_features]\n", + " \"\"\"\n", + " to_replace = []\n", + "\n", + " for full_name, layer in module.named_sublayers(include_self=False):\n", + " if isinstance(layer, nn.Linear):\n", + " parts = full_name.split('.')\n", + " parent_path, child_name = parts[:-1], parts[-1]\n", + " parent = _get_parent_by_path(module, parent_path) if parent_path else module\n", + "\n", + " in_f = layer.weight.shape[0] # I\n", + " out_f = layer.weight.shape[1] # O\n", + "\n", + " if mode == 'be':\n", + " new_layer = LinearBE(in_f, out_f, k)\n", + " with paddle.no_grad():\n", + " # 拷贝共享主权重（[I,O]）与偏置（[O]）\n", + " assert list(new_layer.weight.shape) == list(layer.weight.shape), \\\n", + " f\"weight shape mismatch: {new_layer.weight.shape} vs {layer.weight.shape}\"\n", + " new_layer.weight.set_value(layer.weight.clone())\n", + " if layer.bias is not None and new_layer.bias_e is not None:\n", + " b = layer.bias.reshape([1, -1]).tile([k, 1]) # (K, O)\n", + " assert list(new_layer.bias_e.shape) == list(b.shape), \\\n", + " f\"bias shape mismatch: {new_layer.bias_e.shape} vs {b.shape}\"\n", + " new_layer.bias_e.set_value(b)\n", + " else: # 'packed'\n", + " new_layer = NLinear(k, in_f, out_f, bias=layer.bias is not None)\n", + " with paddle.no_grad():\n", + " # 每个 pack 共享同一权重初值: 原 (I,O) -> (K,I,O)\n", + " w = layer.weight.unsqueeze(0).tile([k, 1, 1]) # (K,I,O)\n", + " assert list(new_layer.weight.shape) == list(w.shape), \\\n", + " f\"packed weight shape mismatch: {new_layer.weight.shape} vs {w.shape}\"\n", + " new_layer.weight.set_value(w)\n", + " if layer.bias is not None and new_layer.bias_e is not None:\n", + " b = layer.bias.unsqueeze(0).tile([k, 1]) # (K,O)\n", + " assert list(new_layer.bias_e.shape) == list(b.shape), \\\n", + " f\"packed bias shape mismatch: {new_layer.bias_e.shape} vs {b.shape}\"\n", + " new_layer.bias_e.set_value(b)\n", + "\n", + " to_replace.append((parent, child_name, new_layer))\n", + "\n", + " # 正式替换\n", + " for parent, child_name, new_layer in to_replace:\n", + " if hasattr(parent, child_name):\n", + " setattr(parent, child_name, new_layer)\n", + " else:\n", + " sub_layers = getattr(parent, \"_sub_layers\", None)\n", + " if sub_layers is None or child_name not in sub_layers:\n", + " raise AttributeError(f\"Cannot set sublayer '{child_name}' under '{type(parent).__name__}'\")\n", + " parent._sub_layers[child_name] = new_layer\n", + "\n", + "# ---------- TabM 特征提取器 --------------------------------------------------\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " \"\"\"\n", + " arch_type: 'plain' | 'tabm' | 'tabm-mini' | 'tabm-packed'\n", + " 返回：\n", + " - reduce=True → (B,H)\n", + " - reduce=False → (B,K,H)\n", + " \"\"\"\n", + " def __init__(self,\n", + " num_features: int,\n", + " arch_type: Literal['plain', 'tabm', 'tabm-mini', 'tabm-packed']='tabm',\n", + " k: int = 32,\n", + " backbone_cfg: Optional[dict] = None,\n", + " reduce: bool = True):\n", + " super().__init__()\n", + " if arch_type == 'plain':\n", + " k = 1\n", + " self.k = k\n", + " self.reduce = reduce\n", + " cfg = backbone_cfg or dict(n_blocks=3, d_hidden=512, dropout=0.1)\n", + " self.backbone = BackboneMLP(**cfg, d_in=num_features)\n", + "\n", + " # --- 插入 Ensemble 逻辑 ---\n", + " if arch_type == 'tabm':\n", + " _replace_linear(self.backbone, k, mode='be')\n", + " self.min_adapter = None\n", + " elif arch_type == 'tabm-mini':\n", + " self.min_adapter = ScaleEnsemble(k, num_features, init='random-signs')\n", + " elif arch_type == 'tabm-packed':\n", + " _replace_linear(self.backbone, k, mode='packed')\n", + " self.min_adapter = None\n", + " else: # plain\n", + " self.min_adapter = None\n", + "\n", + " def forward(self, x_num: paddle.Tensor):\n", + " \"\"\"\n", + " x_num : (B, num_features) – 已完成数值化/标准化\n", + " \"\"\"\n", + " if self.k > 1:\n", + " x = x_num.unsqueeze(1).tile([1, self.k, 1]) # (B,K,D)\n", + " else:\n", + " x = x_num.unsqueeze(1) # (B,1,D)\n", + "\n", + " if self.min_adapter is not None:\n", + " x = self.min_adapter(x) # (B,K,D)\n", + "\n", + " features = self.backbone(x) # (B,K,H)\n", + " if self.reduce:\n", + " return features.mean(axis=1) # (B,H)\n", + " return features # (B,K,H)\n", + "\n", + "# ---------------- Quick check ----------------\n", + "if __name__ == '__main__':\n", + " paddle.seed(123)\n", + " B, D = 8, 30\n", + " x = paddle.randn([B, D])\n", + " # 1) 标准 TabM（BatchEnsemble 替换）\n", + " fe1 = TabMFeatureExtractor(D, arch_type='tabm', k=16, reduce=True)\n", + " out1 = fe1(x)\n", + " print('TabM-BE features:', list(out1.shape)) # (B, H)\n", + "\n", + " # 2) tabm-mini（只做 rank-1 缩放）\n", + " fe2 = TabMFeatureExtractor(D, arch_type='tabm-mini', k=16, reduce=False)\n", + " out2 = fe2(x)\n", + " print('TabM-mini features:', list(out2.shape)) # (B, K, H)\n", + "\n", + " # 3) tabm-packed（Packed NLinear）\n", + " fe3 = TabMFeatureExtractor(D, arch_type='tabm-packed', k=8, reduce=True)\n", + " out3 = fe3(x)\n", + " print('TabM-packed features:', list(out3.shape)) # (B, H)\n", + "\n", + " # 4) plain（无集成基线）\n", + " fe4 = TabMFeatureExtractor(D, arch_type='plain', k=1, reduce=True)\n", + " out4 = fe4(x)\n", + " print('Plain features:', list(out4.shape)) # (B, H)\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "TCF5hoiWF-N4", + "outputId": "df4ea66c-1716-4357-fa58-fb5f8f57d183" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "TabM-BE features: [8, 512]\n", + "TabM-mini features: [8, 16, 512]\n", + "TabM-packed features: [8, 512]\n", + "Plain features: [8, 512]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Literal, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "\n", + "\n", + "# ====== 数据集（示例：合成数据）========================================\n", + "class ToyMultiLabelDataset(Dataset):\n", + " \"\"\"\n", + " 返回:\n", + " x_num: float32, 形状 (D,)\n", + " y: float32, 形状 (4,) —— 多标签 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, d: int, seed: int = 123):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.X = rng.normal(size=(n, d)).astype('float32')\n", + " # 随机生成 4 个线性规则 + 噪声，得到多标签\n", + " W = rng.normal(size=(d, 4))\n", + " logits = self.X @ W + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.Y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " return self.X[idx], self.Y[idx]\n", + "\n", + " def __len__(self) -> int:\n", + " return len(self.X)\n", + "\n", + "# ====== 模型：特征抽取 + 多标签头 ======================================\n", + "class MultiLabelClassifier(nn.Layer):\n", + " def __init__(self, num_features: int, num_labels: int = 4,\n", + " arch_type: str = 'tabm', k: int = 16,\n", + " backbone_cfg: Optional[dict] = None):\n", + " super().__init__()\n", + " self.fe = TabMFeatureExtractor(\n", + " num_features=num_features,\n", + " arch_type=arch_type,\n", + " k=k,\n", + " backbone_cfg=backbone_cfg,\n", + " reduce=True\n", + " )\n", + " # 推断隐藏维度（若你的 TabM 有属性可读，直接使用；否则手动传入）\n", + " d_hidden = getattr(self.fe, \"d_hidden\", (backbone_cfg or dict(d_hidden=512))[\"d_hidden\"])\n", + " self.head = nn.Linear(d_hidden, num_labels)\n", + "\n", + " def forward(self, x_num: paddle.Tensor) -> paddle.Tensor:\n", + " # x_num: (B, D)\n", + " h = self.fe(x_num) # (B, H)\n", + " logits = self.head(h) # (B, 4)\n", + " return logits\n", + "\n", + "# ====== 评价指标：F1、AP 等 ==============================================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " \"\"\"\n", + " y_true: (N, C) 0/1\n", + " y_pred: (N, C) 0/1\n", + " 返回: per_class F1, macro-F1, micro-F1\n", + " \"\"\"\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + "\n", + " # per-class\n", + " for c in range(C):\n", + " yt = y_true[:, c]\n", + " yp = y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + "\n", + " macro_f1 = float(np.mean(f1_c))\n", + "\n", + " # micro\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " \"\"\"\n", + " 简易版 micro-AP(AUCPR)：在 0~1 阈值上扫一遍，近似计算 PR 曲线下面积\n", + " \"\"\"\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " # 按 recall 升序进行梯形积分\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " auc_pr = np.trapz(precision, recall)\n", + " return float(auc_pr)\n", + "\n", + "# ====== 训练/验证循环 =====================================================\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None,\n", + " device: str = 'gpu' if paddle.is_compiled_with_cuda() else 'cpu'):\n", + " model.train()\n", + " total_loss = 0.0\n", + " total_batches = 0\n", + " for x, y in loader:\n", + " x = x.astype('float32')\n", + " y = y.astype('float32')\n", + " logits = model(x)\n", + " # BCE with logits（支持 pos_weight）\n", + " if pos_weight is not None:\n", + " loss = F.binary_cross_entropy_with_logits(logits, y, pos_weight=pos_weight)\n", + " else:\n", + " loss = F.binary_cross_entropy_with_logits(logits, y)\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss)\n", + " total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5):\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x, y in loader:\n", + " x = x.astype('float32'); y = y.astype('float32')\n", + " logits = model(x) # (B,4)\n", + " loss = F.binary_cross_entropy_with_logits(logits, y)\n", + " prob = F.sigmoid(logits).numpy() # (B,4)\n", + " ys.append(y.numpy())\n", + " ps.append(prob)\n", + " total_loss += float(loss)\n", + " total_batches += 1\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + "\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " avg_loss = total_loss / max(1, total_batches)\n", + " metrics = {\n", + " \"loss\": avg_loss,\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + " return metrics\n", + "\n", + "# ====== 主函数：跑通一个最小示例 ===========================================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + " # 配置\n", + " D = 30 # 数值特征维度\n", + " C = 4 # 多标签数\n", + " N_train, N_val = 5000, 1000\n", + " batch_size = 128\n", + " epochs = 5\n", + " lr = 3e-4\n", + "\n", + " # 数据\n", + " train_ds = ToyMultiLabelDataset(N_train, D, seed=42)\n", + " val_ds = ToyMultiLabelDataset(N_val, D, seed=233)\n", + " train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, drop_last=False)\n", + " val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False, drop_last=False)\n", + "\n", + " # 类别不平衡（可选）：按训练集估计每个标签的正例比例，构造 pos_weight\n", + " y_train = np.vstack([y for _, y in train_ds])\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3) # (4,)\n", + " # 经典做法：pos_weight = (N_neg / N_pos) = (1-p)/p\n", + " pos_weight_np = (1.0 - pos_ratio) / pos_ratio\n", + " pos_weight = paddle.to_tensor(pos_weight_np.astype('float32')) # (4,)\n", + "\n", + " # 模型\n", + " backbone_cfg = dict(n_blocks=3, d_hidden=512, dropout=0.1)\n", + " model = MultiLabelClassifier(num_features=D, num_labels=C,\n", + " arch_type='tabm', k=16,\n", + " backbone_cfg=backbone_cfg)\n", + "\n", + " optimizer = paddle.optimizer.Adam(learning_rate=lr, parameters=model.parameters())\n", + "\n", + " # 训练\n", + " best_macro_f1, best_state = -1.0, None\n", + " for ep in range(1, epochs + 1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " # 记录最佳\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best_state = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best_state is not None:\n", + " model.set_state_dict(best_state)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qOJBZjSyGpSA", + "outputId": "c23c05d7-0f63-406f-fbcc-66ea0c3065fc" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/tmp/ipython-input-1539235308.py:108: DeprecationWarning: `trapz` is deprecated. Use `trapezoid` instead, or one of the numerical integration functions in `scipy.integrate`.\n", + " auc_pr = np.trapz(precision, recall)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Epoch 01] train_loss=0.4427 | val_loss=1.5854 | macro_f1=0.4728 | micro_f1=0.4734 | per_class_f1=[0.5263158082962036, 0.5373737215995789, 0.4471057951450348, 0.38046795129776] | micro_AP=0.4584\n", + "[Epoch 02] train_loss=0.1538 | val_loss=2.9504 | macro_f1=0.4818 | micro_f1=0.4837 | per_class_f1=[0.553903341293335, 0.5406504273414612, 0.44742268323898315, 0.38532111048698425] | micro_AP=0.4769\n", + "[Epoch 03] train_loss=0.1057 | val_loss=3.7486 | macro_f1=0.4919 | micro_f1=0.4941 | per_class_f1=[0.5516605377197266, 0.568965494632721, 0.4606299102306366, 0.38624873757362366] | micro_AP=0.4722\n", + "[Epoch 04] train_loss=0.0917 | val_loss=4.2100 | macro_f1=0.4762 | micro_f1=0.4774 | per_class_f1=[0.5183752179145813, 0.557729959487915, 0.43551796674728394, 0.3932472765445709] | micro_AP=0.4652\n", + "[Epoch 05] train_loss=0.0732 | val_loss=4.7243 | macro_f1=0.4810 | micro_f1=0.4820 | per_class_f1=[0.5422138571739197, 0.5443425178527832, 0.4589178264141083, 0.3785425126552582] | micro_AP=0.4507\n", + "Loaded best state with macro_f1=0.4919\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Literal, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "from paddle.vision.models import resnet18\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 2048):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== 简化版 TabM（占位，可换成你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " \"\"\"占位实现：MLP → (B, H)。可直接替换为你修好的 TabM。\"\"\"\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + "\n", + " def forward(self, x_num: paddle.Tensor): # (B, 424)\n", + " return self.net(x_num) # (B, H)\n", + "\n", + "# ====================== ResNet18 特征抽取（逐帧） ======================\n", + "class ResNet18FrameEncoder(nn.Layer):\n", + " \"\"\"将 ResNet18 改为 20 通道输入；输出每帧 512 维特征。\"\"\"\n", + " def __init__(self, in_channels: int = 20):\n", + " super().__init__()\n", + " self.backbone = resnet18(pretrained=False)\n", + " # 改首层卷积为 20 通道\n", + " self.backbone.conv1 = nn.Conv2D(in_channels, 64, kernel_size=7, stride=2, padding=3, bias_attr=False)\n", + " # 去掉分类头 fc，保留到 avgpool\n", + " self.avgpool = self.backbone.avgpool # AdaptiveAvgPool2D(1)\n", + " # 记录下游维度\n", + " self.out_dim = 512\n", + "\n", + " def forward(self, x): # x: (B*T, C=20, H=20, W=20)\n", + " m = self.backbone\n", + " x = m.conv1(x); x = m.bn1(x); x = F.relu(x); x = m.maxpool(x)\n", + " x = m.layer1(x); x = m.layer2(x); x = m.layer3(x); x = m.layer4(x)\n", + " x = self.avgpool(x) # (B*T, 512, 1, 1)\n", + " x = paddle.flatten(x, 1) # (B*T, 512)\n", + " return x\n", + "\n", + "# ====================== 时序 Transformer 编码器 ======================\n", + "class TemporalTransformer(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=4, dim_feedforward=1024, dropout=0.1, max_len=1024):\n", + " super().__init__()\n", + " enc_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead,\n", + " dim_feedforward=dim_feedforward,\n", + " dropout=dropout, activation='relu')\n", + " self.encoder = nn.TransformerEncoder(enc_layer, num_layers=num_layers)\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " x = self.pos(x)\n", + " # Paddle 的 Transformer 期望 (T, B, D)\n", + " x = paddle.transpose(x, [1, 0, 2]) # (T,B,D)\n", + " z = self.encoder(x) # (T,B,D)\n", + " z = paddle.transpose(z, [1, 0, 2]) # (B,T,D)\n", + " return z\n", + "\n", + "# ====================== 多头注意力（支持 q from A, kv from B） ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.ln = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, q, kv):\n", + " \"\"\"\n", + " q: (B, Nq, D)\n", + " kv: (B, Nk, D)\n", + " return: (B, Nq, D) # 残差 + LN\n", + " \"\"\"\n", + " B, Nq, D = q.shape\n", + " Nk = kv.shape[1]\n", + "\n", + " q_lin = self.Wq(q) # (B,Nq,D)\n", + " k_lin = self.Wk(kv) # (B,Nk,D)\n", + " v_lin = self.Wv(kv) # (B,Nk,D)\n", + "\n", + " def split_heads(t): # (B,N,Heads,dh)\n", + " return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0, 2, 1, 3])\n", + "\n", + " qh = split_heads(q_lin) # (B,H,Nq,dh)\n", + " kh = split_heads(k_lin) # (B,H,Nk,dh)\n", + " vh = split_heads(v_lin) # (B,H,Nk,dh)\n", + "\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head) # (B,H,Nq,Nk)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh) # (B,H,Nq,dh)\n", + "\n", + " ctx = ctx.transpose([0, 2, 1, 3]).reshape([B, Nq, D]) # (B,Nq,D)\n", + " out = self.proj(ctx)\n", + " out = self.drop(out)\n", + " # 残差 + LN\n", + " return self.ln(out + q)\n", + "\n", + "# ====================== 融合头（双向 Cross-Attn） ======================\n", + "class BiModalCrossFusion(nn.Layer):\n", + " \"\"\"\n", + " 输入：\n", + " video_seq: (B, T, D) —— Transformer 后的视频序列\n", + " tabm_tok: (B, D) —— TabM token\n", + " 过程：\n", + " v_token = mean(video_seq)\n", + " v' = CrossAttn(q=v_token[1], kv=tabm_token[1])\n", + " t' = CrossAttn(q=tabm_token[1], kv=video_seq[T])\n", + " fuse = concat([v', t']) → MLP\n", + " \"\"\"\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(\n", + " nn.Linear(2 * d_model, fuse_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " )\n", + " self.out_dim = fuse_hidden\n", + "\n", + " def forward(self, video_seq, tabm_tok):\n", + " B, T, D = video_seq.shape\n", + " # 池化出视频 token\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + "\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok) # (B,1,D)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq) # (B,1,D)\n", + "\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1) # (B,1,2D)\n", + " fused = fused.squeeze(1) # (B,2D)\n", + " return self.fuse(fused) # (B, F)\n", + "\n", + "# ====================== 总模型 ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=8, n_trans_layers=4, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4):\n", + " super().__init__()\n", + " # A: 逐帧 ResNet18\n", + " self.frame_encoder = ResNet18FrameEncoder(in_channels=vid_channels) # (B*T,512)\n", + " # A: 时序 Transformer\n", + " self.temporal = TemporalTransformer(d_model=d_model,\n", + " nhead=nhead,\n", + " num_layers=n_trans_layers,\n", + " dim_feedforward=trans_ff,\n", + " dropout=dropout,\n", + " max_len=vid_frames)\n", + " # B: TabM（或替换为你的 TabM）\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model) # 对齐到 d_model\n", + " # 融合：双向 Cross-Attention\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " def forward(self, x_video, x_vec):\n", + " \"\"\"\n", + " x_video: (B, T, C=20, H=20, W=20)\n", + " x_vec: (B, 424)\n", + " \"\"\"\n", + " B, T, C, H, W = x_video.shape\n", + " # ---- A: 逐帧 ResNet ----\n", + " xvt = x_video.reshape([B * T, C, H, W]) # (B*T, C, H, W)\n", + " f_frame = self.frame_encoder(xvt) # (B*T, 512)\n", + " f_seq = f_frame.reshape([B, T, -1]) # (B, T, 512)\n", + " # ---- A: 时序 Transformer ----\n", + " z_vid = self.temporal(f_seq) # (B, T, 512)\n", + " # ---- B: TabM 特征 ----\n", + " z_tabm = self.tabm(x_vec) # (B, H_tabm)\n", + " z_tabm = self.tabm_proj(z_tabm) # (B, 512)\n", + " # ---- Cross-Attention 融合 ----\n", + " fused = self.fusion(z_vid, z_tabm) # (B, 512)\n", + " # ---- 分类 ----\n", + " logits = self.head(fused) # (B, 4)\n", + " return logits\n", + "\n", + "# ====================== 指标与训练循环（与前一致） ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " return float(np.trapz(precision, recall))\n", + "\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None):\n", + " model.train()\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if pos_weight is not None:\n", + " loss = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " else:\n", + " loss = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss); total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5):\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " loss = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " prob = F.sigmoid(logits).numpy()\n", + " ys.append(y.numpy()); ps.append(prob)\n", + " total_loss += float(loss); total_batches += 1\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\n", + " \"loss\": total_loss / max(1, total_batches),\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + "\n", + "# ====================== 合成数据集（可替换为真实数据） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=365, C=20, H=20, W=20)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n\n", + " # 按 (n, T, C, H, W)\n", + " self.video = rng.normal(size=(n, 36, 20, 20, 20)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + "\n", + " # 造标签：对视频先在 H/W 上均值，再在 T 上均值 → (n, C=20)\n", + " vid_hw = self.video.mean(axis=(3, 4)) # (n, T, C)\n", + " vid_avg = vid_hw.mean(axis=1) # (n, C)\n", + "\n", + " # 线性映射到 4 个标签\n", + " Wv = rng.normal(size=(20, 4)) # C→4\n", + " Wt = rng.normal(size=(424, 4)) # 424→4\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " x_vid = self.video[idx] # (T,C,H,W)\n", + " x_vec = self.vec[idx] # (424,)\n", + " y = self.y[idx] # (4,)\n", + " return x_vid, x_vec, y\n", + "\n", + " def __len__(self):\n", + " return self.n\n", + "\n", + "# ====================== 训练入口（可直接运行） ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + " # 数据\n", + " train_ds = ToyTwoModalDataset(n=64, seed=42) # 注意：真实训练建议更大数据与多卡\n", + " val_ds = ToyTwoModalDataset(n=32, seed=233)\n", + " # 自定义 collate：让视频变成 (B,T,C,H,W)\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)), # (B,T,C,H,W)\n", + " paddle.to_tensor(np.stack(vecs, 0)), # (B,424)\n", + " paddle.to_tensor(np.stack(ys, 0))) # (B,4)\n", + " train_loader = DataLoader(train_ds, batch_size=2, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=2, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32')) # (4,)\n", + "\n", + " # 模型\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " vec_dim=424,\n", + " d_model=512, nhead=2, n_trans_layers=2, trans_ff=1024, # 可调\n", + " tabm_hidden=512, dropout=0.1,\n", + " num_labels=4\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用：小 epoch）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 3+1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best is not None:\n", + " model.set_state_dict(best)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "X7-O1-LZLHB2", + "outputId": "124d09eb-3e79-4622-f0b3-b2bf7eb11d60" + }, + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/tmp/ipython-input-3157620546.py:248: DeprecationWarning: `trapz` is deprecated. Use `trapezoid` instead, or one of the numerical integration functions in `scipy.integrate`.\n", + " return float(np.trapz(precision, recall))\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Epoch 01] train_loss=1.1304 | val_loss=1.0465 | macro_f1=0.3765 | micro_f1=0.5079 | per_class_f1=[0.5, 0.7450980544090271, 0.260869562625885, 0.0] | micro_AP=0.5393\n", + "[Epoch 02] train_loss=0.7661 | val_loss=0.9369 | macro_f1=0.5812 | micro_f1=0.6173 | per_class_f1=[0.5454545617103577, 0.7599999904632568, 0.6382978558540344, 0.380952388048172] | micro_AP=0.4165\n", + "[Epoch 03] train_loss=0.4050 | val_loss=1.9395 | macro_f1=0.6107 | micro_f1=0.6303 | per_class_f1=[0.5454545617103577, 0.7234042286872864, 0.6938775777816772, 0.47999998927116394] | micro_AP=0.3836\n", + "Loaded best state with macro_f1=0.6107\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "from paddle.vision.models import resnet18\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 2048):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== 简化版 TabM（占位，可换成你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " \"\"\"占位实现：MLP → (B, H)。可直接替换为你修好的 TabM。\"\"\"\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + "\n", + " def forward(self, x_num: paddle.Tensor): # (B, 424)\n", + " return self.net(x_num) # (B, H)\n", + "\n", + "# ====================== ResNet18 特征抽取（逐帧） ======================\n", + "class ResNet18FrameEncoder(nn.Layer):\n", + " \"\"\"将 ResNet18 改为 20 通道输入；输出每帧 512 维特征。\"\"\"\n", + " def __init__(self, in_channels: int = 20):\n", + " super().__init__()\n", + " self.backbone = resnet18(pretrained=False)\n", + " # 改首层卷积为 20 通道\n", + " self.backbone.conv1 = nn.Conv2D(in_channels, 64, kernel_size=7, stride=2, padding=3, bias_attr=False)\n", + " # 去掉分类头 fc，保留到 avgpool\n", + " self.avgpool = self.backbone.avgpool # AdaptiveAvgPool2D(1)\n", + " self.out_dim = 512\n", + "\n", + " def forward(self, x): # x: (B*T, C=20, H=20, W=20)\n", + " m = self.backbone\n", + " x = m.conv1(x); x = m.bn1(x); x = F.relu(x); x = m.maxpool(x)\n", + " x = m.layer1(x); x = m.layer2(x); x = m.layer3(x); x = m.layer4(x)\n", + " x = self.avgpool(x) # (B*T, 512, 1, 1)\n", + " x = paddle.flatten(x, 1) # (B*T, 512)\n", + " return x\n", + "\n", + "# ====================== MoE 基础实现（Top-k，可开关；使用 gather_nd 修复） ======================\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1, act='relu'):\n", + " super().__init__()\n", + " Act = getattr(F, act) if isinstance(act, str) else act\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.act = Act\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(self.act(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self,\n", + " n_experts=8,\n", + " top_k=1,\n", + " d_ff=2048,\n", + " dropout=0.1,\n", + " router_temp=0.5,\n", + " balance_loss_w=0.005,\n", + " entropy_reg_w=-0.005, # 负值→更尖锐\n", + " diversity_w=1e-3,\n", + " sticky_w=0.0,\n", + " sup_router_w=0.0,\n", + " use_gumbel=True):\n", + " self.n_experts = n_experts\n", + " self.top_k = top_k\n", + " self.d_ff = d_ff\n", + " self.dropout = dropout\n", + " self.router_temp = router_temp\n", + " self.balance_loss_w = balance_loss_w\n", + " self.entropy_reg_w = entropy_reg_w\n", + " self.diversity_w = diversity_w\n", + " self.sticky_w = sticky_w\n", + " self.sup_router_w = sup_router_w\n", + " self.use_gumbel = use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"forward(x, domain_id=None) → (y, aux_loss)，支持 (B,T,D) 或 (N,D)\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(cfg.dropout)\n", + "\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u))\n", + " logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + "\n", + " def forward(self, x, domain_id=None):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3:\n", + " B, T, D = orig_shape\n", + " X = x.reshape([B*T, D])\n", + " else:\n", + " X = x\n", + " N, D = X.shape\n", + "\n", + " logits = self.router(X) # (N,E)\n", + " probs = self._router_probs(logits) # (N,E)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1) # (N,k)\n", + "\n", + " # 专家并行输出\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + "\n", + " # === 使用 gather_nd 逐样本选择 top-k 专家 ===\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list = []\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64') # (N,)\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1) # (N,2) [sample, expert]\n", + " picked_i = paddle.gather_nd(all_out, idx_nd) # (N,D)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + "\n", + " # 归一化权重并加权\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (N,k)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + "\n", + " Y = self.drop(Y)\n", + " Y = self.ln(Y + X)\n", + "\n", + " # aux loss\n", + " aux = 0.0\n", + " if self.cfg.balance_loss_w > 0:\n", + " mean_prob = probs.mean(axis=0)\n", + " target = paddle.full_like(mean_prob, 1.0 / self.cfg.n_experts)\n", + " aux = aux + self.cfg.balance_loss_w * F.mse_loss(mean_prob, target)\n", + " if self.cfg.entropy_reg_w != 0.0:\n", + " ent = -paddle.sum(probs * (paddle.log(probs + 1e-9)), axis=1).mean()\n", + " aux = aux + self.cfg.entropy_reg_w * ent\n", + " if (domain_id is not None) and (self.cfg.sup_router_w > 0):\n", + " dom = domain_id.reshape([-1])[:N] % self.cfg.n_experts\n", + " aux = aux + self.cfg.sup_router_w * F.cross_entropy(logits, dom)\n", + " if self.cfg.diversity_w > 0 and self.cfg.n_experts > 1:\n", + " # 用 top-1 硬选择近似每个专家接收的样本\n", + " chosen = F.one_hot(topk_idx[:, 0], num_classes=self.cfg.n_experts).astype('float32') # (N,E)\n", + " denom = chosen.sum(axis=0).clip(min=1.0).unsqueeze(-1)\n", + " means = (all_out * chosen.unsqueeze(-1)).sum(axis=0) / denom # (E,D)\n", + " sims = []\n", + " for i in range(self.cfg.n_experts):\n", + " for j in range(i+1, self.cfg.n_experts):\n", + " si = F.normalize(means[i:i+1], axis=-1)\n", + " sj = F.normalize(means[j:j+1], axis=-1)\n", + " sims.append((si*sj).sum())\n", + " if sims:\n", + " aux = aux + self.cfg.diversity_w * paddle.stack(sims).mean()\n", + "\n", + " if len(orig_shape) == 3:\n", + " Y = Y.reshape([B, T, D])\n", + " return Y, aux\n", + "\n", + "class MoEHead(nn.Layer):\n", + " \"\"\"单 token MoE 头，用于 fused/tabm 投影后的 (B, D)\"\"\"\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " def forward(self, tok, domain_id=None):\n", + " y, aux = self.moe(tok.unsqueeze(1), domain_id=domain_id) # (B,1,D)\n", + " return y.squeeze(1), aux\n", + "\n", + "# ====================== 自定义 Transformer Encoder（FFN 可替换为 MoE） ======================\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.use_moe = use_moe\n", + " self.self_attn = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.ln1 = nn.LayerNorm(d_model)\n", + " self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " )\n", + " self.do2 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " # Self-Attention (pre-norm) —— Paddle MHA 期望 (T,B,D)\n", + " h = self.ln1(x)\n", + " h = paddle.transpose(h, [1, 0, 2]) # (T,B,D)\n", + " sa = self.self_attn(h, h, h) # (T,B,D)\n", + " sa = paddle.transpose(sa, [1, 0, 2]) # (B,T,D)\n", + " x = x + self.do1(sa)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id) # 残差+LN 在 MoE 内部\n", + " else:\n", + " x = x + self.do2(self.ffn(x)) # 残差在这里\n", + " return x, aux\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=1024, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg)\n", + " for _ in range(num_layers)\n", + " ])\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " x = self.pos(x)\n", + " aux_total = 0.0\n", + " for layer in self.layers:\n", + " x, aux = layer(x, domain_id=domain_id)\n", + " aux_total = aux_total + aux\n", + " return x, aux_total\n", + "\n", + "# ====================== 多头注意力（支持 q from A, kv from B） ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.ln = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape\n", + " Nk = kv.shape[1]\n", + " q_lin = self.Wq(q); k_lin = self.Wk(kv); v_lin = self.Wv(kv)\n", + " def split_heads(t):\n", + " return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0, 2, 1, 3])\n", + " qh = split_heads(q_lin); kh = split_heads(k_lin); vh = split_heads(v_lin)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh)\n", + " ctx = ctx.transpose([0, 2, 1, 3]).reshape([B, Nq, D])\n", + " out = self.proj(ctx)\n", + " out = self.drop(out)\n", + " return self.ln(out + q)\n", + "\n", + "# ====================== 融合头（双向 Cross-Attn） ======================\n", + "class BiModalCrossFusion(nn.Layer):\n", + " \"\"\"\n", + " 输入：\n", + " video_seq: (B, T, D) —— Transformer 后的视频序列\n", + " tabm_tok: (B, D) —— TabM token\n", + " \"\"\"\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(\n", + " nn.Linear(2 * d_model, fuse_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " )\n", + " self.out_dim = fuse_hidden\n", + "\n", + " def forward(self, video_seq, tabm_tok):\n", + " B, T, D = video_seq.shape\n", + " # 池化视频时间维得到 token\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok) # (B,1,D)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq) # (B,1,D)\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1) # (B,1,2D)\n", + " fused = fused.squeeze(1) # (B,2D)\n", + " return self.fuse(fused) # (B, F)\n", + "\n", + "# ====================== 总模型（带三个 MoE 开关） ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=2, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " # ===== MoE 开关 =====\n", + " moe_temporal: bool = True, # 时序 Transformer 的 FFN 位置\n", + " moe_fused: bool = False, # 融合 token 上的小型 MoE 头\n", + " moe_tabm: bool = False, # TabM 投影后\n", + " # ===== MoE 超参（可传入自定义） =====\n", + " moe_cfg_temporal: MoEConfig = None,\n", + " moe_cfg_fused: MoEConfig = None,\n", + " moe_cfg_tabm: MoEConfig = None):\n", + " super().__init__()\n", + " # A: 逐帧 ResNet18\n", + " self.frame_encoder = ResNet18FrameEncoder(in_channels=vid_channels)\n", + " # A: 时序 Transformer（可开/关 MoE）\n", + " self.temporal = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers,\n", + " d_ff=trans_ff, dropout=dropout, max_len=vid_frames,\n", + " use_moe=moe_temporal,\n", + " moe_cfg=moe_cfg_temporal or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(trans_ff, 2048), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # B: TabM（或你的 TabM）\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + "\n", + " # 可选：TabM 分支 MoE 头\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=moe_cfg_tabm or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 融合：双向 Cross-Attention\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + "\n", + " # 可选：融合 token MoE 头\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=moe_cfg_fused or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " def forward(self, x_video, x_vec, domain_id=None):\n", + " \"\"\"\n", + " x_video: (B, T, C=20, H=20, W=20)\n", + " x_vec: (B, 424)\n", + " domain_id: (B,) 或 None —— 若有域/季节/站点标签，可传入以做监督路由（可选）\n", + " \"\"\"\n", + " B, T, C, H, W = x_video.shape\n", + " # ---- A: 逐帧 ResNet ----\n", + " xvt = x_video.reshape([B * T, C, H, W]) # (B*T, C, H, W)\n", + " f_frame = self.frame_encoder(xvt) # (B*T, 512)\n", + " f_seq = f_frame.reshape([B, T, -1]) # (B, T, 512)\n", + " # ---- A: 时序 Transformer (可含 MoE) ----\n", + " z_vid, aux_total = self.temporal(f_seq, domain_id=domain_id) # (B,T,512), aux\n", + "\n", + " # ---- B: TabM 特征 ----\n", + " z_tabm = self.tabm(x_vec) # (B, H_tabm)\n", + " z_tabm = self.tabm_proj(z_tabm) # (B, 512)\n", + " if self.moe_tabm:\n", + " z_tabm, aux_t = self.tabm_moe(z_tabm, domain_id=domain_id) # (B,512)\n", + " aux_total = aux_total + aux_t\n", + "\n", + " # ---- Cross-Attention 融合 ----\n", + " fused = self.fusion(z_vid, z_tabm) # (B, 512)\n", + "\n", + " # ---- 融合 MoE 头（可选） ----\n", + " if self.moe_fused:\n", + " fused, aux_f = self.fused_moe(fused, domain_id=domain_id) # (B,512)\n", + " aux_total = aux_total + aux_f\n", + "\n", + " # ---- 分类 ----\n", + " logits = self.head(fused) # (B, 4)\n", + " return logits, aux_total\n", + "\n", + "# ====================== 指标与训练循环（兼容 aux_loss） ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " return float(np.trapz(precision, recall))\n", + "\n", + "LAMBDA_MOE = 0.01 # MoE 辅助损失系数\n", + "\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None):\n", + " model.train()\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, aux = model(x_vid.astype('float32'), x_vec.astype('float32')) # ← 接收 aux\n", + " if pos_weight is not None:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " else:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss = cls + LAMBDA_MOE * aux\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss); total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5):\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, aux = model(x_vid.astype('float32'), x_vec.astype('float32')) # ← 接收 aux\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss = cls + LAMBDA_MOE * aux\n", + " prob = F.sigmoid(logits).numpy()\n", + " ys.append(y.numpy()); ps.append(prob)\n", + " total_loss += float(loss); total_batches += 1\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\n", + " \"loss\": total_loss / max(1, total_batches),\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + "\n", + "# ====================== 合成数据集（可替换为真实数据） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=36, C=20, H=20, W=20)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n\n", + " # 按 (n, T, C, H, W)\n", + " self.video = rng.normal(size=(n, 36, 20, 20, 20)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + "\n", + " # 造标签：对视频先在 H/W 上均值，再在 T 上均值 → (n, C=20)\n", + " vid_hw = self.video.mean(axis=(3, 4)) # (n, T, C)\n", + " vid_avg = vid_hw.mean(axis=1) # (n, C)\n", + "\n", + " # 线性映射到 4 个标签\n", + " Wv = rng.normal(size=(20, 4)) # C→4\n", + " Wt = rng.normal(size=(424, 4)) # 424→4\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " x_vid = self.video[idx] # (T,C,H,W)\n", + " x_vec = self.vec[idx] # (424,)\n", + " y = self.y[idx] # (4,)\n", + " return x_vid, x_vec, y\n", + "\n", + " def __len__(self):\n", + " return self.n\n", + "\n", + "# ====================== 训练入口（可直接运行） ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + " # 数据\n", + " train_ds = ToyTwoModalDataset(n=64, seed=42)\n", + " val_ds = ToyTwoModalDataset(n=32, seed=233)\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)), # (B,T,C,H,W)\n", + " paddle.to_tensor(np.stack(vecs, 0)), # (B,424)\n", + " paddle.to_tensor(np.stack(ys, 0))) # (B,4)\n", + " train_loader = DataLoader(train_ds, batch_size=2, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=2, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32')) # (4,)\n", + "\n", + " # === 构建模型：三处 MoE 开关（默认只开时序 MoE） ===\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " vec_dim=424,\n", + " d_model=512, nhead=2, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1,\n", + " num_labels=4,\n", + " moe_temporal=True, # 开：时序 Transformer 的 FFN 位置\n", + " moe_fused=False, # 关：融合 token MoE 头\n", + " moe_tabm=False # 关：TabM 投影后 MoE\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用：小 epoch）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 3+1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best is not None:\n", + " model.set_state_dict(best)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "K5On1aO5QtFw", + "outputId": "aeecef88-dd57-4133-8156-f3dd9b6e0c90" + }, + "execution_count": 16, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/tmp/ipython-input-3896222078.py:427: DeprecationWarning: `trapz` is deprecated. Use `trapezoid` instead, or one of the numerical integration functions in `scipy.integrate`.\n", + " return float(np.trapz(precision, recall))\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Epoch 01] train_loss=1.0045 | val_loss=0.8427 | macro_f1=0.1364 | micro_f1=0.2609 | per_class_f1=[0.0, 0.0, 0.0, 0.5454545617103577] | micro_AP=0.4593\n", + "[Epoch 02] train_loss=0.6558 | val_loss=1.0130 | macro_f1=0.4563 | micro_f1=0.5342 | per_class_f1=[0.5945945978164673, 0.0, 0.6938775777816772, 0.5365853905677795] | micro_AP=0.5376\n", + "[Epoch 03] train_loss=0.2265 | val_loss=1.3452 | macro_f1=0.4934 | micro_f1=0.5625 | per_class_f1=[0.4000000059604645, 0.7234042286872864, 0.6000000238418579, 0.25] | micro_AP=0.5076\n", + "Loaded best state with macro_f1=0.4934\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "from paddle.vision.models import resnet18\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 2048):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== 简化版 TabM（占位，可换成你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " \"\"\"占位实现：MLP → (B, H)。可直接替换为你修好的 TabM。\"\"\"\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + "\n", + " def forward(self, x_num: paddle.Tensor): # (B, 424)\n", + " return self.net(x_num) # (B, H)\n", + "\n", + "# ====================== ResNet18 特征抽取（逐帧） ======================\n", + "class ResNet18FrameEncoder(nn.Layer):\n", + " \"\"\"将 ResNet18 改为 20 通道输入；输出每帧 512 维特征。\"\"\"\n", + " def __init__(self, in_channels: int = 20):\n", + " super().__init__()\n", + " self.backbone = resnet18(pretrained=False)\n", + " # 改首层卷积为 20 通道\n", + " self.backbone.conv1 = nn.Conv2D(in_channels, 64, kernel_size=7, stride=2, padding=3, bias_attr=False)\n", + " # 去掉分类头 fc，保留到 avgpool\n", + " self.avgpool = self.backbone.avgpool # AdaptiveAvgPool2D(1)\n", + " self.out_dim = 512\n", + "\n", + " def forward(self, x): # x: (B*T, C=20, H=20, W=20)\n", + " m = self.backbone\n", + " x = m.conv1(x); x = m.bn1(x); x = F.relu(x); x = m.maxpool(x)\n", + " x = m.layer1(x); x = m.layer2(x); x = m.layer3(x); x = m.layer4(x)\n", + " x = self.avgpool(x) # (B*T, 512, 1, 1)\n", + " x = paddle.flatten(x, 1) # (B*T, 512)\n", + " return x\n", + "\n", + "# ====================== MoE 基础实现（Top-k，可开关；使用 gather_nd 修复） ======================\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1, act='relu'):\n", + " super().__init__()\n", + " Act = getattr(F, act) if isinstance(act, str) else act\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.act = Act\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(self.act(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self,\n", + " n_experts=8,\n", + " top_k=1,\n", + " d_ff=2048,\n", + " dropout=0.1,\n", + " router_temp=0.5,\n", + " balance_loss_w=0.005,\n", + " entropy_reg_w=-0.005, # 负值→更尖锐\n", + " diversity_w=1e-3,\n", + " sticky_w=0.0,\n", + " sup_router_w=0.0,\n", + " use_gumbel=True):\n", + " self.n_experts = n_experts\n", + " self.top_k = top_k\n", + " self.d_ff = d_ff\n", + " self.dropout = dropout\n", + " self.router_temp = router_temp\n", + " self.balance_loss_w = balance_loss_w\n", + " self.entropy_reg_w = entropy_reg_w\n", + " self.diversity_w = diversity_w\n", + " self.sticky_w = sticky_w\n", + " self.sup_router_w = sup_router_w\n", + " self.use_gumbel = use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"forward(x, domain_id=None) → (y, aux_loss)，支持 (B,T,D) 或 (N,D)\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(cfg.dropout)\n", + "\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u))\n", + " logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + "\n", + " def forward(self, x, domain_id=None):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3:\n", + " B, T, D = orig_shape\n", + " X = x.reshape([B*T, D])\n", + " else:\n", + " X = x\n", + " N, D = X.shape\n", + "\n", + " logits = self.router(X) # (N,E)\n", + " probs = self._router_probs(logits) # (N,E)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1) # (N,k)\n", + "\n", + " # 专家并行输出\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + "\n", + " # === 使用 gather_nd 逐样本选择 top-k 专家 ===\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list = []\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64') # (N,)\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1) # (N,2) [sample, expert]\n", + " picked_i = paddle.gather_nd(all_out, idx_nd) # (N,D)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + "\n", + " # 归一化权重并加权\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (N,k)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + "\n", + " Y = self.drop(Y)\n", + " Y = self.ln(Y + X)\n", + "\n", + " # aux loss\n", + " aux = 0.0\n", + " if self.cfg.balance_loss_w > 0:\n", + " mean_prob = probs.mean(axis=0)\n", + " target = paddle.full_like(mean_prob, 1.0 / self.cfg.n_experts)\n", + " aux = aux + self.cfg.balance_loss_w * F.mse_loss(mean_prob, target)\n", + " if self.cfg.entropy_reg_w != 0.0:\n", + " ent = -paddle.sum(probs * (paddle.log(probs + 1e-9)), axis=1).mean()\n", + " aux = aux + self.cfg.entropy_reg_w * ent\n", + " if (domain_id is not None) and (self.cfg.sup_router_w > 0):\n", + " dom = domain_id.reshape([-1])[:N] % self.cfg.n_experts\n", + " aux = aux + self.cfg.sup_router_w * F.cross_entropy(logits, dom)\n", + " if self.cfg.diversity_w > 0 and self.cfg.n_experts > 1:\n", + " # 用 top-1 硬选择近似每个专家接收的样本\n", + " chosen = F.one_hot(topk_idx[:, 0], num_classes=self.cfg.n_experts).astype('float32') # (N,E)\n", + " denom = chosen.sum(axis=0).clip(min=1.0).unsqueeze(-1)\n", + " means = (all_out * chosen.unsqueeze(-1)).sum(axis=0) / denom # (E,D)\n", + " sims = []\n", + " for i in range(self.cfg.n_experts):\n", + " for j in range(i+1, self.cfg.n_experts):\n", + " si = F.normalize(means[i:i+1], axis=-1)\n", + " sj = F.normalize(means[j:j+1], axis=-1)\n", + " sims.append((si*sj).sum())\n", + " if sims:\n", + " aux = aux + self.cfg.diversity_w * paddle.stack(sims).mean()\n", + "\n", + " if len(orig_shape) == 3:\n", + " Y = Y.reshape([B, T, D])\n", + " return Y, aux\n", + "\n", + "class MoEHead(nn.Layer):\n", + " \"\"\"单 token MoE 头，用于 fused/tabm 投影后的 (B, D)\"\"\"\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " def forward(self, tok, domain_id=None):\n", + " y, aux = self.moe(tok.unsqueeze(1), domain_id=domain_id) # (B,1,D)\n", + " return y.squeeze(1), aux\n", + "\n", + "# ====================== 自定义 Transformer Encoder（FFN 可替换为 MoE） ======================\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.use_moe = use_moe\n", + " self.self_attn = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.ln1 = nn.LayerNorm(d_model)\n", + " self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " )\n", + " self.do2 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " # Self-Attention (pre-norm) —— Paddle MHA 期望 (T,B,D)\n", + " h = self.ln1(x)\n", + " h = paddle.transpose(h, [1, 0, 2]) # (T,B,D)\n", + " sa = self.self_attn(h, h, h) # (T,B,D)\n", + " sa = paddle.transpose(sa, [1, 0, 2]) # (B,T,D)\n", + " x = x + self.do1(sa)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id) # 残差+LN 在 MoE 内部\n", + " else:\n", + " x = x + self.do2(self.ffn(x)) # 残差在这里\n", + " return x, aux\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=1024, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg)\n", + " for _ in range(num_layers)\n", + " ])\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " x = self.pos(x)\n", + " aux_total = 0.0\n", + " for layer in self.layers:\n", + " x, aux = layer(x, domain_id=domain_id)\n", + " aux_total = aux_total + aux\n", + " return x, aux_total\n", + "\n", + "# ====================== 多头注意力（支持 q from A, kv from B） ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.ln = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape\n", + " Nk = kv.shape[1]\n", + " q_lin = self.Wq(q); k_lin = self.Wk(kv); v_lin = self.Wv(kv)\n", + " def split_heads(t):\n", + " return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0, 2, 1, 3])\n", + " qh = split_heads(q_lin); kh = split_heads(k_lin); vh = split_heads(v_lin)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh)\n", + " ctx = ctx.transpose([0, 2, 1, 3]).reshape([B, Nq, D])\n", + " out = self.proj(ctx)\n", + " out = self.drop(out)\n", + " return self.ln(out + q)\n", + "\n", + "# ====================== 融合头（双向 Cross-Attn） ======================\n", + "class BiModalCrossFusion(nn.Layer):\n", + " \"\"\"\n", + " 输入：\n", + " video_seq: (B, T, D) —— Transformer 后的视频序列\n", + " tabm_tok: (B, D) —— TabM token\n", + " \"\"\"\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(\n", + " nn.Linear(2 * d_model, fuse_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " )\n", + " self.out_dim = fuse_hidden\n", + "\n", + " def forward(self, video_seq, tabm_tok):\n", + " B, T, D = video_seq.shape\n", + " # 池化视频时间维得到 token\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok) # (B,1,D)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq) # (B,1,D)\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1) # (B,1,2D)\n", + " fused = fused.squeeze(1) # (B,2D)\n", + " return self.fuse(fused) # (B, F)\n", + "\n", + "# ====================== 总模型（带三个 MoE 开关） ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=2, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " # ===== MoE 开关 =====\n", + " moe_temporal: bool = True, # 时序 Transformer 的 FFN 位置\n", + " moe_fused: bool = False, # 融合 token 上的小型 MoE 头\n", + " moe_tabm: bool = False, # TabM 投影后\n", + " # ===== MoE 超参（可传入自定义） =====\n", + " moe_cfg_temporal: MoEConfig = None,\n", + " moe_cfg_fused: MoEConfig = None,\n", + " moe_cfg_tabm: MoEConfig = None):\n", + " super().__init__()\n", + " # A: 逐帧 ResNet18\n", + " self.frame_encoder = ResNet18FrameEncoder(in_channels=vid_channels)\n", + " # A: 时序 Transformer（可开/关 MoE）\n", + " self.temporal = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers,\n", + " d_ff=trans_ff, dropout=dropout, max_len=vid_frames,\n", + " use_moe=moe_temporal,\n", + " moe_cfg=moe_cfg_temporal or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(trans_ff, 2048), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # B: TabM（或你的 TabM）\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + "\n", + " # 可选：TabM 分支 MoE 头\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=moe_cfg_tabm or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 融合：双向 Cross-Attention\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + "\n", + " # 可选：融合 token MoE 头\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=moe_cfg_fused or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " # --- 新增：编码函数，导出融合前的 512 维特征（用于检索库） ---\n", + " def encode(self, x_video, x_vec, domain_id=None):\n", + " \"\"\"返回融合后的 512 维 token（分类头之前的表示），不经过最终 Linear。\"\"\"\n", + " B, T, C, H, W = x_video.shape\n", + " xvt = x_video.reshape([B * T, C, H, W])\n", + " f_frame = self.frame_encoder(xvt) # (B*T, 512)\n", + " f_seq = f_frame.reshape([B, T, -1]) # (B, T, 512)\n", + " z_vid, _ = self.temporal(f_seq, domain_id=domain_id) # (B,T,512)\n", + " z_tabm = self.tabm(x_vec)\n", + " z_tabm = self.tabm_proj(z_tabm) # (B,512)\n", + " if self.moe_tabm:\n", + " z_tabm, _ = self.tabm_moe(z_tabm, domain_id=domain_id)\n", + " fused = self.fusion(z_vid, z_tabm) # (B,512)\n", + " if self.moe_fused:\n", + " fused, _ = self.fused_moe(fused, domain_id=domain_id)\n", + " return fused # (B,512)\n", + "\n", + " def forward(self, x_video, x_vec, domain_id=None):\n", + " fused = self.encode(x_video, x_vec, domain_id=domain_id) # (B,512)\n", + " logits = self.head(fused) # (B,4)\n", + " # 为了兼容旧接口，这里返回的 aux 为 0（MoE 的 aux 已在 temporal/tabm_moe/fused_moe 内部求和并丢弃）\n", + " # 如果你想把 MoE 的 aux 在训练里也加入，可把 encode 拆回 forward 的各步并返回累积 aux。\n", + " aux_placeholder = paddle.to_tensor(0.0, dtype='float32')\n", + " return logits, aux_placeholder\n", + "\n", + "# ====================== 指标与训练循环（兼容 aux_loss） ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " return float(np.trapz(precision, recall))\n", + "\n", + "LAMBDA_MOE = 0.0 # 这里的 forward 返回 aux=0（如需把 MoE aux 算进去，可按上一版做法）\n", + "\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None):\n", + " model.train()\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if pos_weight is not None:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " else:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss = cls # + LAMBDA_MOE * aux # 此版本不叠加 MoE aux\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss); total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "# ====================== 检索增强：构建训练库 & 测试时融合 ======================\n", + "class Retriever:\n", + " \"\"\"\n", + " 检索库：\n", + " - keys: 训练集的特征 (N,D) —— 取模型 encode() 的融合特征\n", + " - labels: 训练集的标签 (N,C)\n", + " 推理：\n", + " - 给定测试特征 (B,D)，计算与 keys 的相似度，取 top-k\n", + " - 得到邻居标签的加权均值 p_knn，按 alpha 融合到模型概率\n", + " \"\"\"\n", + " def __init__(self, sim_metric: str = 'cos', k: int = 8, alpha: float = 0.3, tau: float = 1.0):\n", + " \"\"\"\n", + " sim_metric: 'cos' or 'l2'\n", + " k: 近邻数\n", + " alpha: 融合系数，p_final = (1-alpha)*p_model + alpha*p_knn\n", + " tau: 温度（用于 l2 的 softmax(-d/tau) 或 cos 的 softmax(sim/tau)）\n", + " \"\"\"\n", + " assert sim_metric in ['cos', 'l2']\n", + " self.sim_metric = sim_metric\n", + " self.k = k\n", + " self.alpha = alpha\n", + " self.tau = tau\n", + " self.keys = None # (N,D)\n", + " self.labels = None # (N,C)\n", + "\n", + " @paddle.no_grad()\n", + " def build(self, model: nn.Layer, loader: DataLoader):\n", + " model.eval()\n", + " feats, labs = [], []\n", + " for x_vid, x_vec, y in loader:\n", + " f = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,512)\n", + " feats.append(f.numpy())\n", + " labs.append(y.numpy())\n", + " self.keys = paddle.to_tensor(np.concatenate(feats, axis=0)).astype('float32') # (N,D)\n", + " self.labels = paddle.to_tensor(np.concatenate(labs, axis=0)).astype('float32') # (N,C)\n", + " # 预归一化（cos 相似度更快；l2 也可复用）\n", + " self.keys_norm = F.normalize(self.keys, axis=-1)\n", + "\n", + " @paddle.no_grad()\n", + " def query_and_fuse(self, model_probs: paddle.Tensor, test_feat: paddle.Tensor) -> paddle.Tensor:\n", + " \"\"\"\n", + " model_probs: (B,C) —— 模型自身概率（sigmoid后的）\n", + " test_feat: (B,D) —— 模型 encode 导出的融合特征\n", + " return: (B,C) —— 融合后的概率\n", + " \"\"\"\n", + " assert self.keys is not None, \"Call build() before query.\"\n", + " B, D = test_feat.shape\n", + " # 相似度\n", + " if self.sim_metric == 'cos':\n", + " q = F.normalize(test_feat, axis=-1) # (B,D)\n", + " sim = paddle.matmul(q, self.keys_norm, transpose_y=True) # (B,N)\n", + " w = F.softmax(sim / self.tau, axis=-1) # (B,N)\n", + " else: # 'l2'\n", + " # ||q-k||^2 = q^2 + k^2 - 2 q·k\n", + " q2 = paddle.sum(test_feat * test_feat, axis=-1, keepdim=True) # (B,1)\n", + " k2 = paddle.sum(self.keys * self.keys, axis=-1, keepdim=True).transpose([1,0]) # (1,N)\n", + " dot = paddle.matmul(test_feat, self.keys, transpose_y=True) # (B,N)\n", + " dist2 = q2 + k2 - 2.0 * dot # (B,N)\n", + " w = F.softmax(-dist2 / self.tau, axis=-1) # (B,N)\n", + "\n", + " # 取 top-k（可选：先 topk 再归一化，避免长尾干扰）\n", + " topk_val, topk_idx = paddle.topk(w, k=min(self.k, w.shape[1]), axis=-1) # (B,k)\n", + " # gather labels\n", + " N, C = self.labels.shape\n", + " b_idx = paddle.arange(B, dtype='int64').unsqueeze(-1).tile([1, topk_val.shape[1]]) # (B,k)\n", + " # 先 gather 权重对应的 labels\n", + " picked_labels = paddle.gather(self.labels, topk_idx.reshape([-1]), axis=0) # (B*k, C)\n", + " picked_labels = picked_labels.reshape([B, -1, C]) # (B,k,C)\n", + " w_norm = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (B,k)\n", + " p_knn = paddle.sum(picked_labels * w_norm.unsqueeze(-1), axis=1) # (B,C)\n", + "\n", + " # 概率融合\n", + " p_final = (1.0 - self.alpha) * model_probs + self.alpha * p_knn\n", + " return p_final.clip(1e-6, 1-1e-6)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5,\n", + " retriever: Optional[Retriever] = None):\n", + " \"\"\"\n", + " 若 retriever 不为 None：在测试时做 kNN 检索并与模型概率融合。\n", + " \"\"\"\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " prob = F.sigmoid(logits) # (B,C)\n", + "\n", + " if retriever is not None:\n", + " feat = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,D)\n", + " prob = retriever.query_and_fuse(prob, feat) # (B,C)\n", + "\n", + " loss = F.binary_cross_entropy(prob, y.astype('float32')) # 用概率计算 eval loss\n", + " ys.append(y.numpy()); ps.append(prob.numpy())\n", + " total_loss += float(loss); total_batches += 1\n", + "\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\n", + " \"loss\": total_loss / max(1, total_batches),\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + "\n", + "# ====================== 合成数据集（可替换为真实数据） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=36, C=20, H=20, W=20)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n\n", + " self.video = rng.normal(size=(n, 36, 20, 20, 20)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + "\n", + " # 造标签：对视频先在 H/W 上均值，再在 T 上均值 → (n, C=20)\n", + " vid_hw = self.video.mean(axis=(3, 4)) # (n, T, C)\n", + " vid_avg = vid_hw.mean(axis=1) # (n, C)\n", + " Wv = rng.normal(size=(20, 4)) # C→4\n", + " Wt = rng.normal(size=(424, 4)) # 424→4\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + "\n", + " def __len__(self):\n", + " return self.n\n", + "\n", + "# ====================== 训练入口（可直接运行） ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + " # 数据\n", + " train_ds = ToyTwoModalDataset(n=128, seed=42)\n", + " val_ds = ToyTwoModalDataset(n=32, seed=233)\n", + "\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)), # (B,T,C,H,W)\n", + " paddle.to_tensor(np.stack(vecs, 0)), # (B,424)\n", + " paddle.to_tensor(np.stack(ys, 0))) # (B,4)\n", + "\n", + " train_loader = DataLoader(train_ds, batch_size=4, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=4, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32')) # (4,)\n", + "\n", + " # === 模型（MoE 开关按需） ===\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=36,\n", + " vec_dim=424,\n", + " d_model=512, nhead=2, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1,\n", + " num_labels=4,\n", + " moe_temporal=True, # FFN 位置 MoE\n", + " moe_fused=False, # 融合处 MoE\n", + " moe_tabm=False # TabM 处 MoE\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 3+1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5, retriever=None)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best is not None:\n", + " model.set_state_dict(best)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n", + "\n", + " # === 构建检索库（使用训练集） ===\n", + " retr = Retriever(sim_metric='cos', k=8, alpha=0.3, tau=0.5) # 可改 'l2'\n", + " retr.build(model, DataLoader(train_ds, batch_size=8, shuffle=False, collate_fn=collate_fn))\n", + "\n", + " # === 测试时启用检索增强 ===\n", + " val_metrics_knn = evaluate(model, val_loader, threshold=0.5, retriever=retr)\n", + " print(f\"[RkNN] val_loss={val_metrics_knn['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics_knn['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics_knn['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics_knn['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics_knn['micro_AP']:.4f}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "u6m8tTnwSSYC", + "outputId": "b7bfaa2f-71fa-4945-c8b4-b4f586fd0fdf" + }, + "execution_count": 17, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/paddle/nn/layer/norm.py:818: UserWarning: When training, we now always track global mean and variance.\n", + " warnings.warn(\n", + "/tmp/ipython-input-2448755935.py:419: DeprecationWarning: `trapz` is deprecated. Use `trapezoid` instead, or one of the numerical integration functions in `scipy.integrate`.\n", + " return float(np.trapz(precision, recall))\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Epoch 01] train_loss=0.8949 | val_loss=0.7680 | macro_f1=0.4306 | micro_f1=0.5255 | per_class_f1=[0.5454545617103577, 0.5405405163764954, 0.6363636255264282, 0.0] | micro_AP=0.4448\n", + "[Epoch 02] train_loss=0.5957 | val_loss=0.9367 | macro_f1=0.4620 | micro_f1=0.5414 | per_class_f1=[0.4444444477558136, 0.7450980544090271, 0.5333333611488342, 0.125] | micro_AP=0.5133\n", + "[Epoch 03] train_loss=0.3141 | val_loss=1.1780 | macro_f1=0.3715 | micro_f1=0.4158 | per_class_f1=[0.125, 0.5, 0.6000000238418579, 0.260869562625885] | micro_AP=0.5041\n", + "Loaded best state with macro_f1=0.4620\n", + "[RkNN] val_loss=0.8189 | macro_f1=0.4900 | micro_f1=0.5390 | per_class_f1=[0.4000000059604645, 0.7599999904632568, 0.5142857432365417, 0.2857142984867096] | micro_AP=0.4978\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 4096):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== 简化版 TabM（占位，可换你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + " def forward(self, x_num: paddle.Tensor): # (B, 424)\n", + " return self.net(x_num) # (B, H)\n", + "\n", + "# ====================== 3D ResNet-18 体数据特征抽取 ======================\n", + "class BasicBlock3D(nn.Layer):\n", + " expansion = 1\n", + " def __init__(self, in_planes, planes, stride=(1,1,1), downsample=None):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv3D(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(planes)\n", + " self.relu = nn.ReLU()\n", + " self.conv2 = nn.Conv3D(planes, planes, kernel_size=3, stride=1, padding=1, bias_attr=False)\n", + " self.bn2 = nn.BatchNorm3D(planes)\n", + " self.downsample = downsample\n", + " def forward(self, x):\n", + " identity = x\n", + " out = self.relu(self.bn1(self.conv1(x)))\n", + " out = self.bn2(self.conv2(out))\n", + " if self.downsample is not None:\n", + " identity = self.downsample(x)\n", + " out = self.relu(out + identity)\n", + " return out\n", + "\n", + "class ResNet3D(nn.Layer):\n", + " def __init__(self, block, layers, in_channels=20, base_width=64):\n", + " super().__init__()\n", + " self.in_planes = base_width\n", + " # 只空间下采样，保留较细 D 维\n", + " self.conv1 = nn.Conv3D(in_channels, self.in_planes,\n", + " kernel_size=(3,7,7), stride=(1,2,2),\n", + " padding=(1,3,3), bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(self.in_planes)\n", + " self.relu = nn.ReLU()\n", + " self.maxpool = nn.MaxPool3D(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))\n", + " self.layer1 = self._make_layer(block, base_width, layers[0], stride=(1,1,1))\n", + " self.layer2 = self._make_layer(block, base_width*2, layers[1], stride=(2,2,2)) # D/H/W /2\n", + " self.layer3 = self._make_layer(block, base_width*4, layers[2], stride=(2,2,2))\n", + " self.layer4 = self._make_layer(block, base_width*8, layers[3], stride=(2,2,2))\n", + " self.out_dim = base_width*8 # 512\n", + " self.pool = nn.AdaptiveAvgPool3D(output_size=1)\n", + "\n", + " def _make_layer(self, block, planes, blocks, stride=(1,1,1)):\n", + " downsample = None\n", + " if stride != (1,1,1) or self.in_planes != planes * block.expansion:\n", + " downsample = nn.Sequential(\n", + " nn.Conv3D(self.in_planes, planes * block.expansion, kernel_size=1, stride=stride, bias_attr=False),\n", + " nn.BatchNorm3D(planes * block.expansion),\n", + " )\n", + " layers = [block(self.in_planes, planes, stride=stride, downsample=downsample)]\n", + " self.in_planes = planes * block.expansion\n", + " for _ in range(1, blocks):\n", + " layers.append(block(self.in_planes, planes))\n", + " return nn.Sequential(*layers)\n", + "\n", + " def forward(self, x): # x: (B, C, D, H, W)\n", + " x = self.relu(self.bn1(self.conv1(x)))\n", + " x = self.maxpool(x)\n", + " x = self.layer1(x) # (B, 64, D, H/4, W/4)\n", + " x = self.layer2(x) # (B, 128, D/2, H/8, W/8)\n", + " x = self.layer3(x) # (B, 256, D/4, H/16, W/16)\n", + " x = self.layer4(x) # (B, 512, D/8, H/32, W/32)\n", + " x = self.pool(x) # (B, 512, 1,1,1)\n", + " x = paddle.flatten(x, 1) # (B, 512)\n", + " return x\n", + "\n", + "class Volume3DEncoder(nn.Layer):\n", + " \"\"\"\n", + " 3D ResNet-18 over (D,H,W) for each time step.\n", + " 输入单帧体数据: (B, C=20, D=24, H=20, W=20) → 输出 (B, 512)\n", + " \"\"\"\n", + " def __init__(self, in_channels: int = 20, base: int = 64, dropout: float = 0.0):\n", + " super().__init__()\n", + " self.backbone = ResNet3D(BasicBlock3D, layers=[2,2,2,2], in_channels=in_channels, base_width=base)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.out_dim = self.backbone.out_dim # 512\n", + " def forward(self, x): # x: (B, C, D, H, W)\n", + " x = self.backbone(x) # (B,512)\n", + " x = self.drop(x)\n", + " return x\n", + "\n", + "# ====================== MoE（Top-k；gather_nd 选择专家） ======================\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1, act='relu'):\n", + " super().__init__()\n", + " Act = getattr(F, act) if isinstance(act, str) else act\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.act = Act\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(self.act(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self,\n", + " n_experts=8,\n", + " top_k=1,\n", + " d_ff=2048,\n", + " dropout=0.1,\n", + " router_temp=0.5,\n", + " balance_loss_w=0.005,\n", + " entropy_reg_w=-0.005,\n", + " diversity_w=1e-3,\n", + " sticky_w=0.0,\n", + " sup_router_w=0.0,\n", + " use_gumbel=True):\n", + " self.n_experts = n_experts\n", + " self.top_k = top_k\n", + " self.d_ff = d_ff\n", + " self.dropout = dropout\n", + " self.router_temp = router_temp\n", + " self.balance_loss_w = balance_loss_w\n", + " self.entropy_reg_w = entropy_reg_w\n", + " self.diversity_w = diversity_w\n", + " self.sticky_w = sticky_w\n", + " self.sup_router_w = sup_router_w\n", + " self.use_gumbel = use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"forward(x, domain_id=None) → (y, aux_loss)，支持 (B,T,D) 或 (N,D)\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(cfg.dropout)\n", + "\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u))\n", + " logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + "\n", + " def forward(self, x, domain_id=None):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3:\n", + " B, T, D = orig_shape\n", + " X = x.reshape([B*T, D])\n", + " else:\n", + " X = x\n", + " N, D = X.shape\n", + "\n", + " logits = self.router(X) # (N,E)\n", + " probs = self._router_probs(logits) # (N,E)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1) # (N,k)\n", + "\n", + " # 并行专家\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + "\n", + " # gather_nd 逐样本取 top-k\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list = []\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64') # (N,)\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1) # (N,2)\n", + " picked_i = paddle.gather_nd(all_out, idx_nd) # (N,D)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + "\n", + " # 加权\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (N,k)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + "\n", + " # 残差+归一\n", + " Y = self.drop(Y)\n", + " Y = self.ln(Y + X)\n", + "\n", + " # 辅助损失\n", + " aux = 0.0\n", + " if self.cfg.balance_loss_w > 0:\n", + " mean_prob = probs.mean(axis=0)\n", + " target = paddle.full_like(mean_prob, 1.0 / self.cfg.n_experts)\n", + " aux = aux + self.cfg.balance_loss_w * F.mse_loss(mean_prob, target)\n", + " if self.cfg.entropy_reg_w != 0.0:\n", + " ent = -paddle.sum(probs * (paddle.log(probs + 1e-9)), axis=1).mean()\n", + " aux = aux + self.cfg.entropy_reg_w * ent\n", + " if (domain_id is not None) and (self.cfg.sup_router_w > 0):\n", + " dom = domain_id.reshape([-1])[:N] % self.cfg.n_experts\n", + " aux = aux + self.cfg.sup_router_w * F.cross_entropy(logits, dom)\n", + " if self.cfg.diversity_w > 0 and self.cfg.n_experts > 1:\n", + " chosen = F.one_hot(topk_idx[:, 0], num_classes=self.cfg.n_experts).astype('float32') # (N,E)\n", + " denom = chosen.sum(axis=0).clip(min=1.0).unsqueeze(-1)\n", + " means = (all_out * chosen.unsqueeze(-1)).sum(axis=0) / denom # (E,D)\n", + " sims = []\n", + " for i in range(self.cfg.n_experts):\n", + " for j in range(i+1, self.cfg.n_experts):\n", + " si = F.normalize(means[i:i+1], axis=-1)\n", + " sj = F.normalize(means[j:j+1], axis=-1)\n", + " sims.append((si*sj).sum())\n", + " if sims:\n", + " aux = aux + self.cfg.diversity_w * paddle.stack(sims).mean()\n", + "\n", + " if len(orig_shape) == 3:\n", + " Y = Y.reshape([B, T, D])\n", + " return Y, aux\n", + "\n", + "class MoEHead(nn.Layer):\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " def forward(self, tok, domain_id=None):\n", + " y, aux = self.moe(tok.unsqueeze(1), domain_id=domain_id) # (B,1,D)\n", + " return y.squeeze(1), aux\n", + "\n", + "# ====================== 自定义 Transformer Encoder（FFN 可替换 MoE） ======================\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.use_moe = use_moe\n", + " self.self_attn = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.ln1 = nn.LayerNorm(d_model)\n", + " self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " )\n", + " self.do2 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, domain_id=None): # (B,T,D)\n", + " h = self.ln1(x)\n", + " h = paddle.transpose(h, [1, 0, 2]) # (T,B,D)\n", + " sa = self.self_attn(h, h, h) # (T,B,D)\n", + " sa = paddle.transpose(sa, [1, 0, 2]) # (B,T,D)\n", + " x = x + self.do1(sa)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id)\n", + " else:\n", + " x = x + self.do2(self.ffn(x))\n", + " return x, aux\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=4096, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg)\n", + " for _ in range(num_layers)\n", + " ])\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " x = self.pos(x)\n", + " aux_total = 0.0\n", + " for layer in self.layers:\n", + " x, aux = layer(x, domain_id=domain_id)\n", + " aux_total = aux_total + aux\n", + " return x, aux_total\n", + "\n", + "# ====================== Cross-Attention 融合 ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.ln = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape\n", + " q_lin = self.Wq(q); k_lin = self.Wk(kv); v_lin = self.Wv(kv)\n", + " def split_heads(t):\n", + " return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0, 2, 1, 3])\n", + " qh = split_heads(q_lin); kh = split_heads(k_lin); vh = split_heads(v_lin)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh)\n", + " ctx = ctx.transpose([0, 2, 1, 3]).reshape([B, Nq, D])\n", + " out = self.proj(ctx)\n", + " out = self.drop(out)\n", + " return self.ln(out + q)\n", + "\n", + "class BiModalCrossFusion(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(\n", + " nn.Linear(2 * d_model, fuse_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " )\n", + " self.out_dim = fuse_hidden\n", + "\n", + " def forward(self, video_seq, tabm_tok):\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok) # (B,1,D)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq) # (B,1,D)\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1) # (B,1,2D)\n", + " fused = fused.squeeze(1) # (B,2D)\n", + " return self.fuse(fused) # (B, F)\n", + "\n", + "# ====================== 总模型 ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=365, depth_n=24,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " # MoE 开关\n", + " moe_temporal: bool = True,\n", + " moe_fused: bool = False,\n", + " moe_tabm: bool = False,\n", + " # MoE 超参\n", + " moe_cfg_temporal: MoEConfig = None,\n", + " moe_cfg_fused: MoEConfig = None,\n", + " moe_cfg_tabm: MoEConfig = None):\n", + " super().__init__()\n", + " # A: 逐帧 3D ResNet18\n", + " self.vol_encoder = Volume3DEncoder(in_channels=vid_channels, dropout=dropout) # (B*T,512)\n", + " # A: 时序 Transformer（可 MoE）\n", + " self.temporal = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers,\n", + " d_ff=trans_ff, dropout=dropout, max_len=vid_frames,\n", + " use_moe=moe_temporal,\n", + " moe_cfg=moe_cfg_temporal or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(2048, trans_ff), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # B: TabM\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + "\n", + " # 可选：TabM 分支 MoE 头\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=moe_cfg_tabm or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 融合\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + "\n", + " # 可选：融合 token MoE 头\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=moe_cfg_fused or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " self.vid_frames = vid_frames\n", + " self.depth_n = depth_n\n", + "\n", + " # 导出融合前 512 表示（用于检索库）\n", + " def encode(self, x_video, x_vec, domain_id=None):\n", + " \"\"\"\n", + " x_video: (B, T, C=20, H=20, W=20, N=24)\n", + " x_vec: (B, 424)\n", + " \"\"\"\n", + " B, T, C, H, W, N = x_video.shape\n", + " assert N == self.depth_n, f\"N mismatch: got {N}, expect {self.depth_n}\"\n", + " # 逐帧 3D 编码： (B*T, C, D=N, H, W)\n", + " xvt = x_video.transpose([0,1,2,5,3,4]).reshape([B*T, C, N, H, W])\n", + " f_frame = self.vol_encoder(xvt) # (B*T, 512)\n", + " f_seq = f_frame.reshape([B, T, -1]) # (B, T, 512)\n", + " z_vid, _ = self.temporal(f_seq, domain_id=domain_id) # (B,T,512)\n", + " z_tabm = self.tabm(x_vec)\n", + " z_tabm = self.tabm_proj(z_tabm) # (B,512)\n", + " if self.moe_tabm:\n", + " z_tabm, _ = self.tabm_moe(z_tabm, domain_id=domain_id)\n", + " fused = self.fusion(z_vid, z_tabm) # (B,512)\n", + " if self.moe_fused:\n", + " fused, _ = self.fused_moe(fused, domain_id=domain_id)\n", + " return fused\n", + "\n", + " def forward(self, x_video, x_vec, domain_id=None):\n", + " fused = self.encode(x_video, x_vec, domain_id=domain_id) # (B,512)\n", + " logits = self.head(fused) # (B,4)\n", + " aux_placeholder = paddle.to_tensor(0.0, dtype='float32')\n", + " return logits, aux_placeholder\n", + "\n", + "# ====================== 指标与训练循环 ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " return float(np.trapz(precision, recall))\n", + "\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None):\n", + " model.train()\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if pos_weight is not None:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " else:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss = cls\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss); total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "# ====================== 检索增强（cos / l2；k 邻居软加权；概率融合） ======================\n", + "class Retriever:\n", + " def __init__(self, sim_metric: str = 'cos', k: int = 8, alpha: float = 0.3, tau: float = 0.5):\n", + " assert sim_metric in ['cos', 'l2']\n", + " self.sim_metric = sim_metric\n", + " self.k = k\n", + " self.alpha = alpha\n", + " self.tau = tau\n", + " self.keys = None # (N,D)\n", + " self.labels = None # (N,C)\n", + "\n", + " @paddle.no_grad()\n", + " def build(self, model: nn.Layer, loader: DataLoader):\n", + " model.eval()\n", + " feats, labs = [], []\n", + " for x_vid, x_vec, y in loader:\n", + " f = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,512)\n", + " feats.append(f.numpy())\n", + " labs.append(y.numpy())\n", + " self.keys = paddle.to_tensor(np.concatenate(feats, axis=0)).astype('float32') # (N,D)\n", + " self.labels = paddle.to_tensor(np.concatenate(labs, axis=0)).astype('float32') # (N,C)\n", + " self.keys_norm = F.normalize(self.keys, axis=-1)\n", + "\n", + " @paddle.no_grad()\n", + " def query_and_fuse(self, model_probs: paddle.Tensor, test_feat: paddle.Tensor) -> paddle.Tensor:\n", + " assert self.keys is not None, \"build() must be called first.\"\n", + " B, D = test_feat.shape\n", + " if self.sim_metric == 'cos':\n", + " q = F.normalize(test_feat, axis=-1)\n", + " sim = paddle.matmul(q, self.keys_norm, transpose_y=True) # (B,N)\n", + " w = F.softmax(sim / self.tau, axis=-1)\n", + " else:\n", + " q2 = paddle.sum(test_feat * test_feat, axis=-1, keepdim=True) # (B,1)\n", + " k2 = paddle.sum(self.keys * self.keys, axis=-1, keepdim=True).transpose([1,0]) # (1,N)\n", + " dot = paddle.matmul(test_feat, self.keys, transpose_y=True) # (B,N)\n", + " dist2 = q2 + k2 - 2.0 * dot # (B,N)\n", + " w = F.softmax(-dist2 / self.tau, axis=-1)\n", + "\n", + " topk_val, topk_idx = paddle.topk(w, k=min(self.k, w.shape[1]), axis=-1) # (B,k)\n", + " picked_labels = paddle.gather(self.labels, topk_idx.reshape([-1]), axis=0) # (B*k, C)\n", + " C = self.labels.shape[1]\n", + " picked_labels = picked_labels.reshape([B, -1, C]) # (B,k,C)\n", + " w_norm = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (B,k)\n", + " p_knn = paddle.sum(picked_labels * w_norm.unsqueeze(-1), axis=1) # (B,C)\n", + "\n", + " p_final = (1.0 - self.alpha) * model_probs + self.alpha * p_knn\n", + " return p_final.clip(1e-6, 1-1e-6)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5,\n", + " retriever: Optional[Retriever] = None):\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " prob = F.sigmoid(logits) # (B,C)\n", + " if retriever is not None:\n", + " feat = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,512)\n", + " prob = retriever.query_and_fuse(prob, feat)\n", + " loss = F.binary_cross_entropy(prob, y.astype('float32'))\n", + " ys.append(y.numpy()); ps.append(prob.numpy())\n", + " total_loss += float(loss); total_batches += 1\n", + "\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\n", + " \"loss\": total_loss / max(1, total_batches),\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + "\n", + "# ====================== ToyDataset（T=365, N=24） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=365, C=20, H=20, W=20, N=24)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0, T: int = 365, C: int = 20, H: int = 20, W: int = 20, N: int = 24):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n\n", + " self.T, self.C, self.H, self.W, self.N = T, C, H, W, N\n", + " # (n, T, C, H, W, N)\n", + " self.video = rng.normal(size=(n, T, C, H, W, N)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + "\n", + " # 造标签：对视频先在 H/W/N 上均值，再在 T 上均值 → (n, C)\n", + " vid_hwn = self.video.mean(axis=(3, 4, 5)) # (n, T, C)\n", + " vid_avg = vid_hwn.mean(axis=1) # (n, C)\n", + "\n", + " Wv = rng.normal(size=(C, 4))\n", + " Wt = rng.normal(size=(424, 4))\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + " def __len__(self):\n", + " return self.n\n", + "\n", + "# ====================== 训练入口 ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + "\n", + " # 数据\n", + " T, C, H, W, N = 365, 20, 20, 20, 24\n", + " train_ds = ToyTwoModalDataset(n=32, seed=42, T=T, C=C, H=H, W=W, N=N)\n", + " val_ds = ToyTwoModalDataset(n=16, seed=233, T=T, C=C, H=H, W=W, N=N)\n", + "\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)), # (B,T,C,H,W,N)\n", + " paddle.to_tensor(np.stack(vecs, 0)), # (B,424)\n", + " paddle.to_tensor(np.stack(ys, 0))) # (B,4)\n", + "\n", + " # T=365 + 3D 卷积较吃内存，示例用小 batch\n", + " train_loader = DataLoader(train_ds, batch_size=1, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=1, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32')) # (4,)\n", + "\n", + " # 模型\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=C, vid_h=H, vid_w=W, vid_frames=T, depth_n=N,\n", + " vec_dim=424,\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1,\n", + " num_labels=4,\n", + " moe_temporal=True, # 推荐开启（FFN 位置 MoE）\n", + " moe_fused=False,\n", + " moe_tabm=False\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 2+1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5, retriever=None)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best is not None:\n", + " model.set_state_dict(best)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n", + "\n", + " # === 构建检索库（用训练集） ===\n", + " retr = Retriever(sim_metric='cos', k=8, alpha=0.3, tau=0.5) # 可改 'l2'\n", + " retr.build(model, DataLoader(train_ds, batch_size=1, shuffle=False, collate_fn=collate_fn))\n", + "\n", + " # === 测试时启用检索增强 ===\n", + " val_metrics_knn = evaluate(model, val_loader, threshold=0.5, retriever=retr)\n", + " print(f\"[RkNN] val_loss={val_metrics_knn['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics_knn['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics_knn['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics_knn['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics_knn['micro_AP']:.4f}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "a85yLOn8jaJU", + "outputId": "acabc284-ffc9-4f25-ca2b-46e7971e2234" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md\n", + " warnings.warn(warning_message)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 4096):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + "\n", + " def forward(self, x): # x: (B, T, D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== 简化版 TabM（占位，可换你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + " def forward(self, x_num: paddle.Tensor): # (B, 424)\n", + " return self.net(x_num) # (B, H)\n", + "\n", + "# ====================== 3D ResNet-18 体数据特征抽取 ======================\n", + "class BasicBlock3D(nn.Layer):\n", + " expansion = 1\n", + " def __init__(self, in_planes, planes, stride=(1,1,1), downsample=None):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv3D(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(planes)\n", + " self.relu = nn.ReLU()\n", + " self.conv2 = nn.Conv3D(planes, planes, kernel_size=3, stride=1, padding=1, bias_attr=False)\n", + " self.bn2 = nn.BatchNorm3D(planes)\n", + " self.downsample = downsample\n", + " def forward(self, x):\n", + " identity = x\n", + " out = self.relu(self.bn1(self.conv1(x)))\n", + " out = self.bn2(self.conv2(out))\n", + " if self.downsample is not None:\n", + " identity = self.downsample(x)\n", + " out = self.relu(out + identity)\n", + " return out\n", + "\n", + "class ResNet3D(nn.Layer):\n", + " def __init__(self, block, layers, in_channels=20, base_width=64):\n", + " super().__init__()\n", + " self.in_planes = base_width\n", + " # 只空间下采样，保留较细 D 维\n", + " self.conv1 = nn.Conv3D(in_channels, self.in_planes,\n", + " kernel_size=(3,7,7), stride=(1,2,2),\n", + " padding=(1,3,3), bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(self.in_planes)\n", + " self.relu = nn.ReLU()\n", + " self.maxpool = nn.MaxPool3D(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))\n", + " self.layer1 = self._make_layer(block, base_width, layers[0], stride=(1,1,1))\n", + " self.layer2 = self._make_layer(block, base_width*2, layers[1], stride=(2,2,2)) # D/H/W /2\n", + " self.layer3 = self._make_layer(block, base_width*4, layers[2], stride=(2,2,2))\n", + " self.layer4 = self._make_layer(block, base_width*8, layers[3], stride=(2,2,2))\n", + " self.out_dim = base_width*8 # 512\n", + " self.pool = nn.AdaptiveAvgPool3D(output_size=1)\n", + "\n", + " def _make_layer(self, block, planes, blocks, stride=(1,1,1)):\n", + " downsample = None\n", + " if stride != (1,1,1) or self.in_planes != planes * block.expansion:\n", + " downsample = nn.Sequential(\n", + " nn.Conv3D(self.in_planes, planes * block.expansion, kernel_size=1, stride=stride, bias_attr=False),\n", + " nn.BatchNorm3D(planes * block.expansion),\n", + " )\n", + " layers = [block(self.in_planes, planes, stride=stride, downsample=downsample)]\n", + " self.in_planes = planes * block.expansion\n", + " for _ in range(1, blocks):\n", + " layers.append(block(self.in_planes, planes))\n", + " return nn.Sequential(*layers)\n", + "\n", + " def forward(self, x): # x: (B, C, D, H, W)\n", + " x = self.relu(self.bn1(self.conv1(x)))\n", + " x = self.maxpool(x)\n", + " x = self.layer1(x) # (B, 64, D, H/4, W/4)\n", + " x = self.layer2(x) # (B, 128, D/2, H/8, W/8)\n", + " x = self.layer3(x) # (B, 256, D/4, H/16, W/16)\n", + " x = self.layer4(x) # (B, 512, D/8, H/32, W/32)\n", + " x = self.pool(x) # (B, 512, 1,1,1)\n", + " x = paddle.flatten(x, 1) # (B, 512)\n", + " return x\n", + "\n", + "class Volume3DEncoder(nn.Layer):\n", + " \"\"\"\n", + " 3D ResNet-18 over (D,H,W) for each time step.\n", + " 输入单帧体数据: (B, C=20, D=24, H=20, W=20) → 输出 (B, 512)\n", + " \"\"\"\n", + " def __init__(self, in_channels: int = 20, base: int = 64, dropout: float = 0.0):\n", + " super().__init__()\n", + " self.backbone = ResNet3D(BasicBlock3D, layers=[2,2,2,2], in_channels=in_channels, base_width=base)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.out_dim = self.backbone.out_dim # 512\n", + " def forward(self, x): # x: (B, C, D, H, W)\n", + " x = self.backbone(x) # (B,512)\n", + " x = self.drop(x)\n", + " return x\n", + "\n", + "# ====================== MoE（Top-k；gather_nd 选择专家） ======================\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1, act='relu'):\n", + " super().__init__()\n", + " Act = getattr(F, act) if isinstance(act, str) else act\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.act = Act\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(self.act(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self,\n", + " n_experts=8,\n", + " top_k=1,\n", + " d_ff=2048,\n", + " dropout=0.1,\n", + " router_temp=0.5,\n", + " balance_loss_w=0.005,\n", + " entropy_reg_w=-0.005,\n", + " diversity_w=1e-3,\n", + " sticky_w=0.0,\n", + " sup_router_w=0.0,\n", + " use_gumbel=True):\n", + " self.n_experts = n_experts\n", + " self.top_k = top_k\n", + " self.d_ff = d_ff\n", + " self.dropout = dropout\n", + " self.router_temp = router_temp\n", + " self.balance_loss_w = balance_loss_w\n", + " self.entropy_reg_w = entropy_reg_w\n", + " self.diversity_w = diversity_w\n", + " self.sticky_w = sticky_w\n", + " self.sup_router_w = sup_router_w\n", + " self.use_gumbel = use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"forward(x, domain_id=None) → (y, aux_loss)，支持 (B,T,D) 或 (N,D)\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(cfg.dropout)\n", + "\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u))\n", + " logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + "\n", + " def forward(self, x, domain_id=None):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3:\n", + " B, T, D = orig_shape\n", + " X = x.reshape([B*T, D])\n", + " else:\n", + " X = x\n", + " N, D = X.shape\n", + "\n", + " logits = self.router(X) # (N,E)\n", + " probs = self._router_probs(logits) # (N,E)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1) # (N,k)\n", + "\n", + " # 并行专家\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + "\n", + " # gather_nd 逐样本取 top-k\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list = []\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64') # (N,)\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1) # (N,2)\n", + " picked_i = paddle.gather_nd(all_out, idx_nd) # (N,D)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + "\n", + " # 加权\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (N,k)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + "\n", + " # 残差+归一\n", + " Y = self.drop(Y)\n", + " Y = self.ln(Y + X)\n", + "\n", + " # 辅助损失\n", + " aux = 0.0\n", + " if self.cfg.balance_loss_w > 0:\n", + " mean_prob = probs.mean(axis=0)\n", + " target = paddle.full_like(mean_prob, 1.0 / self.cfg.n_experts)\n", + " aux = aux + self.cfg.balance_loss_w * F.mse_loss(mean_prob, target)\n", + " if self.cfg.entropy_reg_w != 0.0:\n", + " ent = -paddle.sum(probs * (paddle.log(probs + 1e-9)), axis=1).mean()\n", + " aux = aux + self.cfg.entropy_reg_w * ent\n", + " if (domain_id is not None) and (self.cfg.sup_router_w > 0):\n", + " dom = domain_id.reshape([-1])[:N] % self.cfg.n_experts\n", + " aux = aux + self.cfg.sup_router_w * F.cross_entropy(logits, dom)\n", + " if self.cfg.diversity_w > 0 and self.cfg.n_experts > 1:\n", + " chosen = F.one_hot(topk_idx[:, 0], num_classes=self.cfg.n_experts).astype('float32') # (N,E)\n", + " denom = chosen.sum(axis=0).clip(min=1.0).unsqueeze(-1)\n", + " means = (all_out * chosen.unsqueeze(-1)).sum(axis=0) / denom # (E,D)\n", + " sims = []\n", + " for i in range(self.cfg.n_experts):\n", + " for j in range(i+1, self.cfg.n_experts):\n", + " si = F.normalize(means[i:i+1], axis=-1)\n", + " sj = F.normalize(means[j:j+1], axis=-1)\n", + " sims.append((si*sj).sum())\n", + " if sims:\n", + " aux = aux + self.cfg.diversity_w * paddle.stack(sims).mean()\n", + "\n", + " if len(orig_shape) == 3:\n", + " Y = Y.reshape([B, T, D])\n", + " return Y, aux\n", + "\n", + "class MoEHead(nn.Layer):\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " def forward(self, tok, domain_id=None):\n", + " y, aux = self.moe(tok.unsqueeze(1), domain_id=domain_id) # (B,1,D)\n", + " return y.squeeze(1), aux\n", + "\n", + "# ====================== 自定义 Transformer Encoder（FFN 可替换 MoE） ======================\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.use_moe = use_moe\n", + " self.self_attn = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.ln1 = nn.LayerNorm(d_model)\n", + " self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " )\n", + " self.do2 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, domain_id=None): # (B,T,D)\n", + " h = self.ln1(x)\n", + " h = paddle.transpose(h, [1, 0, 2]) # (T,B,D)\n", + " sa = self.self_attn(h, h, h) # (T,B,D)\n", + " sa = paddle.transpose(sa, [1, 0, 2]) # (B,T,D)\n", + " x = x + self.do1(sa)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id)\n", + " else:\n", + " x = x + self.do2(self.ffn(x))\n", + " return x, aux\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=4096, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg)\n", + " for _ in range(num_layers)\n", + " ])\n", + " def forward(self, x, domain_id=None): # x: (B,T,D)\n", + " x = self.pos(x)\n", + " aux_total = 0.0\n", + " for layer in self.layers:\n", + " x, aux = layer(x, domain_id=domain_id)\n", + " aux_total = aux_total + aux\n", + " return x, aux_total\n", + "\n", + "# ====================== Cross-Attention 融合 ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.ln = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape\n", + " q_lin = self.Wq(q); k_lin = self.Wk(kv); v_lin = self.Wv(kv)\n", + " def split_heads(t):\n", + " return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0, 2, 1, 3])\n", + " qh = split_heads(q_lin); kh = split_heads(k_lin); vh = split_heads(v_lin)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh)\n", + " ctx = ctx.transpose([0, 2, 1, 3]).reshape([B, Nq, D])\n", + " out = self.proj(ctx)\n", + " out = self.drop(out)\n", + " return self.ln(out + q)\n", + "\n", + "class BiModalCrossFusion(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(\n", + " nn.Linear(2 * d_model, fuse_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " )\n", + " self.out_dim = fuse_hidden\n", + "\n", + " def forward(self, video_seq, tabm_tok):\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok) # (B,1,D)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq) # (B,1,D)\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1) # (B,1,2D)\n", + " fused = fused.squeeze(1) # (B,2D)\n", + " return self.fuse(fused) # (B, F)\n", + "\n", + "# ====================== 总模型 ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=365, depth_n=24,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " # MoE 开关\n", + " moe_temporal: bool = True,\n", + " moe_fused: bool = False,\n", + " moe_tabm: bool = False,\n", + " # MoE 超参\n", + " moe_cfg_temporal: MoEConfig = None,\n", + " moe_cfg_fused: MoEConfig = None,\n", + " moe_cfg_tabm: MoEConfig = None):\n", + " super().__init__()\n", + " # A: 逐帧 3D ResNet18\n", + " self.vol_encoder = Volume3DEncoder(in_channels=vid_channels, dropout=dropout) # (B*T,512)\n", + " # A: 时序 Transformer（可 MoE）\n", + " self.temporal = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers,\n", + " d_ff=trans_ff, dropout=dropout, max_len=vid_frames,\n", + " use_moe=moe_temporal,\n", + " moe_cfg=moe_cfg_temporal or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(2048, trans_ff), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # B: TabM\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + "\n", + " # 可选：TabM 分支 MoE 头\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=moe_cfg_tabm or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 融合\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + "\n", + " # 可选：融合 token MoE 头\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=moe_cfg_fused or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " self.vid_frames = vid_frames\n", + " self.depth_n = depth_n\n", + "\n", + " # 导出融合前 512 表示（用于检索库）\n", + " def encode(self, x_video, x_vec, domain_id=None):\n", + " \"\"\"\n", + " x_video: (B, T, C=20, H=20, W=20, N=24)\n", + " x_vec: (B, 424)\n", + " \"\"\"\n", + " B, T, C, H, W, N = x_video.shape\n", + " assert N == self.depth_n, f\"N mismatch: got {N}, expect {self.depth_n}\"\n", + " # 逐帧 3D 编码： (B*T, C, D=N, H, W)\n", + " xvt = x_video.transpose([0,1,2,5,3,4]).reshape([B*T, C, N, H, W])\n", + " f_frame = self.vol_encoder(xvt) # (B*T, 512)\n", + " f_seq = f_frame.reshape([B, T, -1]) # (B, T, 512)\n", + " z_vid, _ = self.temporal(f_seq, domain_id=domain_id) # (B,T,512)\n", + " z_tabm = self.tabm(x_vec)\n", + " z_tabm = self.tabm_proj(z_tabm) # (B,512)\n", + " if self.moe_tabm:\n", + " z_tabm, _ = self.tabm_moe(z_tabm, domain_id=domain_id)\n", + " fused = self.fusion(z_vid, z_tabm) # (B,512)\n", + " if self.moe_fused:\n", + " fused, _ = self.fused_moe(fused, domain_id=domain_id)\n", + " return fused\n", + "\n", + " def forward(self, x_video, x_vec, domain_id=None):\n", + " fused = self.encode(x_video, x_vec, domain_id=domain_id) # (B,512)\n", + " logits = self.head(fused) # (B,4)\n", + " aux_placeholder = paddle.to_tensor(0.0, dtype='float32')\n", + " return logits, aux_placeholder\n", + "\n", + "# ====================== 指标与训练循环 ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " assert y_true.shape == y_pred.shape\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1))\n", + " fp = np.sum((yt == 0) & (yp == 1))\n", + " fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps)\n", + " rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9)\n", + " rec = tp / (tp + fn + 1e-9)\n", + " micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " for t in thresholds:\n", + " y_pred = (y_prob >= t).astype(np.float32)\n", + " tp = np.sum((y_true == 1) & (y_pred == 1))\n", + " fp = np.sum((y_true == 0) & (y_pred == 1))\n", + " fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " p = tp / (tp + fp + 1e-9)\n", + " r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " recall = np.array(recall)[order]\n", + " precision = np.array(precision)[order]\n", + " return float(np.trapz(precision, recall))\n", + "\n", + "def train_one_epoch(model, loader, optimizer,\n", + " pos_weight: Optional[paddle.Tensor] = None,\n", + " clip_grad_norm: Optional[float] = None):\n", + " model.train()\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if pos_weight is not None:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " else:\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'))\n", + " loss = cls\n", + " loss.backward()\n", + " if clip_grad_norm is not None:\n", + " nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)\n", + " optimizer.step()\n", + " optimizer.clear_grad()\n", + " total_loss += float(loss); total_batches += 1\n", + " return total_loss / max(1, total_batches)\n", + "\n", + "# ====================== 检索增强（cos / l2；k 邻居软加权；概率融合） ======================\n", + "class Retriever:\n", + " def __init__(self, sim_metric: str = 'cos', k: int = 8, alpha: float = 0.3, tau: float = 0.5):\n", + " assert sim_metric in ['cos', 'l2']\n", + " self.sim_metric = sim_metric\n", + " self.k = k\n", + " self.alpha = alpha\n", + " self.tau = tau\n", + " self.keys = None # (N,D)\n", + " self.labels = None # (N,C)\n", + "\n", + " @paddle.no_grad()\n", + " def build(self, model: nn.Layer, loader: DataLoader):\n", + " model.eval()\n", + " feats, labs = [], []\n", + " for x_vid, x_vec, y in loader:\n", + " f = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,512)\n", + " feats.append(f.numpy())\n", + " labs.append(y.numpy())\n", + " self.keys = paddle.to_tensor(np.concatenate(feats, axis=0)).astype('float32') # (N,D)\n", + " self.labels = paddle.to_tensor(np.concatenate(labs, axis=0)).astype('float32') # (N,C)\n", + " self.keys_norm = F.normalize(self.keys, axis=-1)\n", + "\n", + " @paddle.no_grad()\n", + " def query_and_fuse(self, model_probs: paddle.Tensor, test_feat: paddle.Tensor) -> paddle.Tensor:\n", + " assert self.keys is not None, \"build() must be called first.\"\n", + " B, D = test_feat.shape\n", + " if self.sim_metric == 'cos':\n", + " q = F.normalize(test_feat, axis=-1)\n", + " sim = paddle.matmul(q, self.keys_norm, transpose_y=True) # (B,N)\n", + " w = F.softmax(sim / self.tau, axis=-1)\n", + " else:\n", + " q2 = paddle.sum(test_feat * test_feat, axis=-1, keepdim=True) # (B,1)\n", + " k2 = paddle.sum(self.keys * self.keys, axis=-1, keepdim=True).transpose([1,0]) # (1,N)\n", + " dot = paddle.matmul(test_feat, self.keys, transpose_y=True) # (B,N)\n", + " dist2 = q2 + k2 - 2.0 * dot # (B,N)\n", + " w = F.softmax(-dist2 / self.tau, axis=-1)\n", + "\n", + " topk_val, topk_idx = paddle.topk(w, k=min(self.k, w.shape[1]), axis=-1) # (B,k)\n", + " picked_labels = paddle.gather(self.labels, topk_idx.reshape([-1]), axis=0) # (B*k, C)\n", + " C = self.labels.shape[1]\n", + " picked_labels = picked_labels.reshape([B, -1, C]) # (B,k,C)\n", + " w_norm = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9) # (B,k)\n", + " p_knn = paddle.sum(picked_labels * w_norm.unsqueeze(-1), axis=1) # (B,C)\n", + "\n", + " p_final = (1.0 - self.alpha) * model_probs + self.alpha * p_knn\n", + " return p_final.clip(1e-6, 1-1e-6)\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5,\n", + " retriever: Optional[Retriever] = None):\n", + " model.eval()\n", + " ys, ps = [], []\n", + " total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " prob = F.sigmoid(logits) # (B,C)\n", + " if retriever is not None:\n", + " feat = model.encode(x_vid.astype('float32'), x_vec.astype('float32')) # (B,512)\n", + " prob = retriever.query_and_fuse(prob, feat)\n", + " loss = F.binary_cross_entropy(prob, y.astype('float32'))\n", + " ys.append(y.numpy()); ps.append(prob.numpy())\n", + " total_loss += float(loss); total_batches += 1\n", + "\n", + " y_true = np.concatenate(ys, axis=0)\n", + " y_prob = np.concatenate(ps, axis=0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\n", + " \"loss\": total_loss / max(1, total_batches),\n", + " \"macro_f1\": macro_f1,\n", + " \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(),\n", + " \"micro_AP\": ap_micro\n", + " }\n", + "\n", + "# ====================== ToyDataset（T=365, N=24） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=365, C=20, H=20, W=20, N=24)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0, T: int = 365, C: int = 20, H: int = 20, W: int = 20, N: int = 24):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n\n", + " self.T, self.C, self.H, self.W, self.N = T, C, H, W, N\n", + " # (n, T, C, H, W, N)\n", + " self.video = rng.normal(size=(n, T, C, H, W, N)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + "\n", + " # 造标签：对视频先在 H/W/N 上均值，再在 T 上均值 → (n, C)\n", + " vid_hwn = self.video.mean(axis=(3, 4, 5)) # (n, T, C)\n", + " vid_avg = vid_hwn.mean(axis=1) # (n, C)\n", + "\n", + " Wv = rng.normal(size=(C, 4))\n", + " Wt = rng.normal(size=(424, 4))\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n, 4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + "\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + " def __len__(self):\n", + " return self.n\n", + "\n", + "# ====================== 训练入口 ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + "\n", + " # 数据\n", + " T, C, H, W, N = 365, 20, 20, 20, 24\n", + " train_ds = ToyTwoModalDataset(n=32, seed=42, T=T, C=C, H=H, W=W, N=N)\n", + " val_ds = ToyTwoModalDataset(n=16, seed=233, T=T, C=C, H=H, W=W, N=N)\n", + "\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)), # (B,T,C,H,W,N)\n", + " paddle.to_tensor(np.stack(vecs, 0)), # (B,424)\n", + " paddle.to_tensor(np.stack(ys, 0))) # (B,4)\n", + "\n", + " # T=365 + 3D 卷积较吃内存，示例用小 batch\n", + " train_loader = DataLoader(train_ds, batch_size=1, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=1, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32')) # (4,)\n", + "\n", + " # 模型\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=C, vid_h=H, vid_w=W, vid_frames=T, depth_n=N,\n", + " vec_dim=424,\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1,\n", + " num_labels=4,\n", + " moe_temporal=True, # 推荐开启（FFN 位置 MoE）\n", + " moe_fused=False,\n", + " moe_tabm=False\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 2+1):\n", + " train_loss = train_one_epoch(model, train_loader, optimizer,\n", + " pos_weight=pos_weight, clip_grad_norm=1.0)\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5, retriever=None)\n", + " print(f\"[Epoch {ep:02d}] train_loss={train_loss:.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + "\n", + " if best is not None:\n", + " model.set_state_dict(best)\n", + " print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n", + "\n", + " # === 构建检索库（用训练集） ===\n", + " retr = Retriever(sim_metric='cos', k=8, alpha=0.3, tau=0.5) # 可改 'l2'\n", + " retr.build(model, DataLoader(train_ds, batch_size=1, shuffle=False, collate_fn=collate_fn))\n", + "\n", + " # === 测试时启用检索增强 ===\n", + " val_metrics_knn = evaluate(model, val_loader, threshold=0.5, retriever=retr)\n", + " print(f\"[RkNN] val_loss={val_metrics_knn['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics_knn['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics_knn['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics_knn['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics_knn['micro_AP']:.4f}\")\n" + ], + "metadata": { + "id": "RwuikckFkM1O" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math\n", + "from typing import Optional, Tuple\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "# 选中第 0 张 GPU；如有多卡改成 'gpu:1' 等\n", + "# paddle.set_device('gpu:0')\n", + "\n", + "# ====================== 工具：正弦位置编码 ======================\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 4096):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + " def forward(self, x): # (B,T,D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ====================== TabM（占位，可换你的实现） ======================\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden),\n", + " nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + " def forward(self, x_num: paddle.Tensor):\n", + " return self.net(x_num)\n", + "\n", + "# ====================== 3D ResNet-18 体数据特征抽取 ======================\n", + "class BasicBlock3D(nn.Layer):\n", + " expansion = 1\n", + " def __init__(self, in_planes, planes, stride=(1,1,1), downsample=None):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv3D(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(planes)\n", + " self.relu = nn.ReLU()\n", + " self.conv2 = nn.Conv3D(planes, planes, kernel_size=3, stride=1, padding=1, bias_attr=False)\n", + " self.bn2 = nn.BatchNorm3D(planes)\n", + " self.downsample = downsample\n", + " def forward(self, x):\n", + " identity = x\n", + " out = self.relu(self.bn1(self.conv1(x)))\n", + " out = self.bn2(self.conv2(out))\n", + " if self.downsample is not None:\n", + " identity = self.downsample(x)\n", + " out = self.relu(out + identity)\n", + " return out\n", + "\n", + "class ResNet3D(nn.Layer):\n", + " def __init__(self, block, layers, in_channels=20, base_width=64):\n", + " super().__init__()\n", + " self.in_planes = base_width\n", + " self.conv1 = nn.Conv3D(in_channels, self.in_planes,\n", + " kernel_size=(3,7,7), stride=(1,2,2),\n", + " padding=(1,3,3), bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(self.in_planes)\n", + " self.relu = nn.ReLU()\n", + " self.maxpool = nn.MaxPool3D(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))\n", + " self.layer1 = self._make_layer(block, base_width, layers[0], stride=(1,1,1))\n", + " self.layer2 = self._make_layer(block, base_width*2, layers[1], stride=(2,2,2))\n", + " self.layer3 = self._make_layer(block, base_width*4, layers[2], stride=(2,2,2))\n", + " self.layer4 = self._make_layer(block, base_width*8, layers[3], stride=(2,2,2))\n", + " self.out_dim = base_width*8 # 512\n", + " self.pool = nn.AdaptiveAvgPool3D(output_size=1)\n", + " def _make_layer(self, block, planes, blocks, stride=(1,1,1)):\n", + " downsample = None\n", + " if stride != (1,1,1) or self.in_planes != planes * block.expansion:\n", + " downsample = nn.Sequential(\n", + " nn.Conv3D(self.in_planes, planes * block.expansion, kernel_size=1, stride=stride, bias_attr=False),\n", + " nn.BatchNorm3D(planes * block.expansion),\n", + " )\n", + " layers = [block(self.in_planes, planes, stride=stride, downsample=downsample)]\n", + " self.in_planes = planes * block.expansion\n", + " for _ in range(1, blocks):\n", + " layers.append(block(self.in_planes, planes))\n", + " return nn.Sequential(*layers)\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.relu(self.bn1(self.conv1(x)))\n", + " x = self.maxpool(x)\n", + " x = self.layer1(x)\n", + " x = self.layer2(x)\n", + " x = self.layer3(x)\n", + " x = self.layer4(x)\n", + " x = self.pool(x) # (B, 512, 1,1,1)\n", + " x = paddle.flatten(x, 1) # (B, 512)\n", + " return x\n", + "\n", + "class Volume3DEncoder(nn.Layer):\n", + " def __init__(self, in_channels: int = 20, base: int = 64, dropout: float = 0.0):\n", + " super().__init__()\n", + " self.backbone = ResNet3D(BasicBlock3D, layers=[2,2,2,2], in_channels=in_channels, base_width=base)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.out_dim = self.backbone.out_dim # 512\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.backbone(x)\n", + " x = self.drop(x)\n", + " return x\n", + "\n", + "# ====================== MoE（Top-k；gather_nd 选择专家） ======================\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1, act='relu'):\n", + " super().__init__()\n", + " Act = getattr(F, act) if isinstance(act, str) else act\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.act = Act\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(self.act(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self,\n", + " n_experts=8, top_k=1, d_ff=2048, dropout=0.1,\n", + " router_temp=0.5, balance_loss_w=0.005, entropy_reg_w=-0.005,\n", + " diversity_w=1e-3, sticky_w=0.0, sup_router_w=0.0, use_gumbel=True):\n", + " self.n_experts = n_experts; self.top_k = top_k; self.d_ff = d_ff; self.dropout = dropout\n", + " self.router_temp = router_temp; self.balance_loss_w = balance_loss_w\n", + " self.entropy_reg_w = entropy_reg_w; self.diversity_w = diversity_w\n", + " self.sticky_w = sticky_w; self.sup_router_w = sup_router_w; self.use_gumbel = use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(cfg.dropout)\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u)); logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + " def forward(self, x, domain_id=None):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3:\n", + " B, T, D = orig_shape; X = x.reshape([B*T, D])\n", + " else:\n", + " X = x\n", + " N, D = X.shape\n", + " logits = self.router(X); probs = self._router_probs(logits)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1)\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list = []\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64')\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1)\n", + " picked_i = paddle.gather_nd(all_out, idx_nd)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1)\n", + " Y = self.drop(Y); Y = self.ln(Y + X)\n", + " aux = 0.0\n", + " if self.cfg.balance_loss_w > 0:\n", + " mean_prob = probs.mean(axis=0)\n", + " target = paddle.full_like(mean_prob, 1.0 / self.cfg.n_experts)\n", + " aux = aux + self.cfg.balance_loss_w * F.mse_loss(mean_prob, target)\n", + " if self.cfg.entropy_reg_w != 0.0:\n", + " ent = -paddle.sum(probs * (paddle.log(probs + 1e-9)), axis=1).mean()\n", + " aux = aux + self.cfg.entropy_reg_w * ent\n", + " if (domain_id is not None) and (self.cfg.sup_router_w > 0):\n", + " dom = domain_id.reshape([-1])[:N] % self.cfg.n_experts\n", + " aux = aux + self.cfg.sup_router_w * F.cross_entropy(logits, dom)\n", + " if self.cfg.diversity_w > 0 and self.cfg.n_experts > 1:\n", + " chosen = F.one_hot(topk_idx[:, 0], num_classes=self.cfg.n_experts).astype('float32')\n", + " denom = chosen.sum(axis=0).clip(min=1.0).unsqueeze(-1)\n", + " means = (all_out * chosen.unsqueeze(-1)).sum(axis=0) / denom\n", + " sims = []\n", + " for i in range(self.cfg.n_experts):\n", + " for j in range(i+1, self.cfg.n_experts):\n", + " si = F.normalize(means[i:i+1], axis=-1)\n", + " sj = F.normalize(means[j:j+1], axis=-1)\n", + " sims.append((si*sj).sum())\n", + " if sims:\n", + " aux = aux + self.cfg.diversity_w * paddle.stack(sims).mean()\n", + " if len(orig_shape) == 3:\n", + " Y = Y.reshape([B, T, D])\n", + " return Y, aux\n", + "\n", + "class MoEHead(nn.Layer):\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " def forward(self, tok, domain_id=None):\n", + " y, aux = self.moe(tok.unsqueeze(1), domain_id=domain_id)\n", + " return y.squeeze(1), aux\n", + "\n", + "# ====================== Self-Attention Transformer（可 MoE） ======================\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.use_moe = use_moe\n", + " self.self_attn = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.ln1 = nn.LayerNorm(d_model); self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " ); self.do2 = nn.Dropout(dropout)\n", + " def forward(self, x, domain_id=None): # (B,T,D)\n", + " h = self.ln1(x)\n", + " h = paddle.transpose(h, [1,0,2])\n", + " sa = self.self_attn(h, h, h)\n", + " sa = paddle.transpose(sa, [1,0,2])\n", + " x = x + self.do1(sa)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id)\n", + " else:\n", + " x = x + self.do2(self.ffn(x))\n", + " return x, aux\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=4096, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout, use_moe=use_moe, moe_cfg=moe_cfg)\n", + " for _ in range(num_layers)\n", + " ])\n", + " def forward(self, x, domain_id=None):\n", + " x = self.pos(x); aux_total = 0.0\n", + " for layer in self.layers:\n", + " x, aux = layer(x, domain_id=domain_id); aux_total += aux\n", + " return x, aux_total\n", + "\n", + "# ====================== AFNO(1D) + MoE FFN ======================\n", + "class AFNO1DLayer(nn.Layer):\n", + " \"\"\"\n", + " 自适应傅里叶算子（时间 1D 版）：\n", + " - 对 (B,T,D) 沿 T 做 rFFT → (B,D,F)\n", + " - 仅保留前 K=modes 个频率，对每个频率在“通道组内”做两层复线性（W1,W2）+ GELU + Softshrink\n", + " - 把频谱其余部分置零 → irFFT → 残差 + Dropout + (可选 LN)\n", + " \"\"\"\n", + " def __init__(self, d_model: int, modes: int = 32, num_blocks: int = 8,\n", + " shrink: float = 0.01, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % num_blocks == 0, \"d_model must be divisible by num_blocks\"\n", + " self.d_model = d_model\n", + " self.modes = modes\n", + " self.num_blocks = num_blocks\n", + " self.block = d_model // num_blocks\n", + " self.shrink = shrink\n", + " # 复权重拆成实/虚：形状 (G, Cb, Cb)\n", + " scale = 1.0 / math.sqrt(self.block)\n", + " def param():\n", + " return nn.initializer.Uniform(-scale, scale)\n", + " self.w1r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=param())\n", + " self.w1i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=param())\n", + " self.w2r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=param())\n", + " self.w2i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=param())\n", + " self.ln = nn.LayerNorm(d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + "\n", + " def _complex_linear(self, xr, xi, Wr, Wi):\n", + " # xr, xi: (B, G, K, Cb); Wr/Wi: (G, Cb, Cb)\n", + " # (a+ib)*(Wr+iWi) = (a@Wr - b@Wi) + i(a@Wi + b@Wr)\n", + " out_r = paddle.matmul(xr, Wr) - paddle.matmul(xi, Wi)\n", + " out_i = paddle.matmul(xr, Wi) + paddle.matmul(xi, Wr)\n", + " return out_r, out_i\n", + "\n", + " def forward(self, x): # x: (B,T,D)\n", + " B, T, D = x.shape\n", + " Kmax = T // 2 + 1\n", + " K = min(self.modes, Kmax)\n", + "\n", + " h = self.ln(x) # PreNorm\n", + " h_td = paddle.transpose(h, [0, 2, 1]) # (B,D,T)\n", + " h_ft = paddle.fft.rfft(h_td) # (B,D,F) complex64\n", + "\n", + " # reshape 通道为 G 组： (B,G,Cb,F)\n", + " h_ft = h_ft.reshape([B, self.num_blocks, self.block, Kmax])\n", + " # 仅前 K 频率： (B,G,Cb,K) → 交换到 (B,G,K,Cb) 方便 matmul\n", + " xk = h_ft[:, :, :, :K].transpose([0,1,3,2])\n", + " xr, xi = paddle.real(xk), paddle.imag(xk) # (B,G,K,Cb)\n", + "\n", + " # 组内两层复线性 + GELU + Softshrink\n", + " yr, yi = self._complex_linear(xr, xi, self.w1r, self.w1i)\n", + " yr = F.gelu(yr); yi = F.gelu(yi)\n", + " # Softshrink（稀疏化）\n", + " # yr = F.softshrink(yr, lambd=self.shrink); yi = F.softshrink(yi, lambd=self.shrink)\n", + " yr = F.softshrink(yr, threshold=self.shrink)\n", + " yi = F.softshrink(yi, threshold=self.shrink)\n", + " yr, yi = self._complex_linear(yr, yi, self.w2r, self.w2i) # (B,G,K,Cb)\n", + "\n", + "\n", + "\n", + "\n", + " # 放回谱： (B,G,K,Cb) → (B,G,Cb,K) → (B,D,K)\n", + " yk = paddle.complex(yr, yi).transpose([0,1,3,2]).reshape([B, D, K])\n", + " out_ft = paddle.zeros([B, D, Kmax], dtype='complex64')\n", + " out_ft[:, :, :K] = yk\n", + "\n", + " # 反变换 & 残差\n", + " out_td = paddle.fft.irfft(out_ft, n=T) # (B,D,T)\n", + " out = paddle.transpose(out_td, [0, 2, 1]) # (B,T,D)\n", + " out = self.drop(out)\n", + " return x + out\n", + "\n", + "class AFNOTransformerFlexible(nn.Layer):\n", + " \"\"\"\n", + " 堆叠若干 AFNO1DLayer；随后接 MoE FFN（与 Self-Attn 分支同构）\n", + " \"\"\"\n", + " def __init__(self, d_model=512, num_layers=2, modes=32, dropout=0.1,\n", + " d_ff=1024, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.layers = nn.LayerList([AFNO1DLayer(d_model, modes=modes, num_blocks=8, shrink=0.01, dropout=dropout)\n", + " for _ in range(num_layers)])\n", + " self.use_moe = use_moe\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(\n", + " nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model),\n", + " )\n", + " self.do = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, domain_id=None): # (B,T,D)\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " aux = 0.0\n", + " if self.use_moe:\n", + " x, aux = self.moe(x, domain_id=domain_id)\n", + " else:\n", + " x = x + self.do(self.ffn(x))\n", + " return x, aux\n", + "\n", + "# ====================== Cross-Attention 融合 ======================\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_head = d_model // nhead; self.nhead = nhead\n", + " self.Wq = nn.Linear(d_model, d_model); self.Wk = nn.Linear(d_model, d_model); self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model); self.drop = nn.Dropout(dropout); self.ln = nn.LayerNorm(d_model)\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape\n", + " def split(t): return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0,2,1,3])\n", + " qh = split(self.Wq(q)); kh = split(self.Wk(kv)); vh = split(self.Wv(kv))\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head)\n", + " attn = F.softmax(scores, axis=-1)\n", + " ctx = paddle.matmul(attn, vh).transpose([0,2,1,3]).reshape([B, Nq, D])\n", + " out = self.drop(self.proj(ctx))\n", + " return self.ln(out + q)\n", + "\n", + "class BiModalCrossFusion(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(nn.Linear(2*d_model, fuse_hidden), nn.ReLU(), nn.Dropout(dropout))\n", + " self.out_dim = fuse_hidden\n", + " def forward(self, video_seq, tabm_tok):\n", + " v_tok = video_seq.mean(axis=1, keepdim=True)\n", + " t_tok = tabm_tok.unsqueeze(1)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq)\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1).squeeze(1)\n", + " return self.fuse(fused)\n", + "\n", + "# ====================== 总模型：Self-Attn + AFNO 并行 ======================\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self,\n", + " # 视频模态\n", + " vid_channels=20, vid_h=20, vid_w=20, vid_frames=365, depth_n=24,\n", + " # 结构化模态\n", + " vec_dim=424,\n", + " # 维度与结构\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " # MoE 开关\n", + " moe_temporal_attn: bool = True,\n", + " moe_temporal_afno: bool = True,\n", + " moe_fused: bool = False,\n", + " moe_tabm: bool = False,\n", + " # AFNO 频率数\n", + " afno_modes: int = 32,\n", + " # MoE 超参\n", + " moe_cfg_temporal_attn: MoEConfig = None,\n", + " moe_cfg_temporal_afno: MoEConfig = None,\n", + " moe_cfg_fused: MoEConfig = None,\n", + " moe_cfg_tabm: MoEConfig = None):\n", + " super().__init__()\n", + " # 逐帧 3D ResNet18\n", + " self.vol_encoder = Volume3DEncoder(in_channels=vid_channels, dropout=dropout)\n", + " # Self-Attention Transformer\n", + " self.trans_attn = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers, d_ff=trans_ff, dropout=dropout,\n", + " max_len=vid_frames, use_moe=moe_temporal_attn,\n", + " moe_cfg=moe_cfg_temporal_attn or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(2048, trans_ff), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # AFNO Transformer（1D）\n", + " self.trans_afno = AFNOTransformerFlexible(\n", + " d_model=d_model, num_layers=n_trans_layers, modes=afno_modes, dropout=dropout,\n", + " d_ff=trans_ff, use_moe=moe_temporal_afno,\n", + " moe_cfg=moe_cfg_temporal_afno or MoEConfig(\n", + " n_experts=8, top_k=1, d_ff=max(2048, trans_ff), router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " )\n", + " )\n", + " # 两路拼接后投回 d_model\n", + " self.video_merge = nn.Linear(2*d_model, d_model)\n", + "\n", + " # TabM\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=moe_cfg_tabm or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 融合\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=moe_cfg_fused or MoEConfig(\n", + " n_experts=6, top_k=1, d_ff=1024, router_temp=0.5,\n", + " balance_loss_w=0.005, entropy_reg_w=-0.005, diversity_w=1e-3\n", + " ))\n", + "\n", + " # 分类头\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + "\n", + " self.vid_frames = vid_frames; self.depth_n = depth_n\n", + "\n", + " # 导出融合前 512 表示（用于检索库）\n", + " def encode(self, x_video, x_vec, domain_id=None):\n", + " \"\"\"\n", + " x_video: (B,T,C,H,W,N) —— N 为体深度（24）\n", + " \"\"\"\n", + " B, T, C, H, W, N = x_video.shape\n", + " assert N == self.depth_n, f\"N mismatch: {N} vs {self.depth_n}\"\n", + " xvt = x_video.transpose([0,1,2,5,3,4]).reshape([B*T, C, N, H, W])\n", + " f_frame = self.vol_encoder(xvt) # (B*T,512)\n", + " seq = f_frame.reshape([B, T, -1]) # (B,T,512)\n", + "\n", + " z_attn, _ = self.trans_attn(seq, domain_id=domain_id) # (B,T,512)\n", + " z_afno, _ = self.trans_afno(seq, domain_id=domain_id) # (B,T,512)\n", + " z_vid = self.video_merge(paddle.concat([z_attn, z_afno], axis=-1)) # (B,T,512)\n", + "\n", + " z_tabm = self.tabm(x_vec); z_tabm = self.tabm_proj(z_tabm) # (B,512)\n", + " if self.moe_tabm:\n", + " z_tabm, _ = self.tabm_moe(z_tabm, domain_id=domain_id)\n", + "\n", + " fused = self.fusion(z_vid, z_tabm) # (B,512)\n", + " if self.moe_fused:\n", + " fused, _ = self.fused_moe(fused, domain_id=domain_id)\n", + " return fused\n", + "\n", + " def forward(self, x_video, x_vec, domain_id=None):\n", + " fused = self.encode(x_video, x_vec, domain_id=domain_id)\n", + " logits = self.head(fused) # (B,4)\n", + " return logits, paddle.to_tensor(0.0, dtype='float32')\n", + "\n", + "# ====================== 简洁指标（可替换为你之前的“全量指标”版本） ======================\n", + "def f1_per_class(y_true: np.ndarray, y_pred: np.ndarray, eps: float = 1e-9) -> Tuple[np.ndarray, float, float]:\n", + " N, C = y_true.shape\n", + " f1_c = np.zeros(C, dtype=np.float32)\n", + " for c in range(C):\n", + " yt, yp = y_true[:, c], y_pred[:, c]\n", + " tp = np.sum((yt == 1) & (yp == 1)); fp = np.sum((yt == 0) & (yp == 1)); fn = np.sum((yt == 1) & (yp == 0))\n", + " prec = tp / (tp + fp + eps); rec = tp / (tp + fn + eps)\n", + " f1_c[c] = 2 * prec * rec / (prec + rec + eps)\n", + " macro_f1 = float(np.mean(f1_c))\n", + " tp = np.sum((y_true == 1) & (y_pred == 1)); fp = np.sum((y_true == 0) & (y_pred == 1)); fn = np.sum((y_true == 1) & (y_pred == 0))\n", + " prec = tp / (tp + fp + 1e-9); rec = tp / (tp + fn + 1e-9); micro_f1 = 2 * prec * rec / (prec + rec + 1e-9)\n", + " return f1_c, macro_f1, float(micro_f1)\n", + "\n", + "def average_precision_micro(y_true: np.ndarray, y_prob: np.ndarray, num_thresholds: int = 101) -> float:\n", + " thresholds = np.linspace(0.0, 1.0, num_thresholds)\n", + " precision, recall = [], []\n", + " yt = y_true.reshape(-1); ps = y_prob.reshape(-1)\n", + " for t in thresholds:\n", + " yp = (ps >= t).astype(np.float32)\n", + " tp = np.sum((yt == 1) & (yp == 1)); fp = np.sum((yt == 0) & (yp == 1)); fn = np.sum((yt == 1) & (yp == 0))\n", + " p = tp / (tp + fp + 1e-9); r = tp / (tp + fn + 1e-9)\n", + " precision.append(p); recall.append(r)\n", + " order = np.argsort(recall)\n", + " return float(np.trapz(np.array(precision)[order], np.array(recall)[order]))\n", + "\n", + "@paddle.no_grad()\n", + "def evaluate(model, loader, threshold: float = 0.5, retriever=None):\n", + " model.eval()\n", + " ys, ps, total_loss, total_batches = [], [], 0.0, 0\n", + " for x_vid, x_vec, y in loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " prob = F.sigmoid(logits)\n", + " if retriever is not None:\n", + " feat = model.encode(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " prob = retriever.query_and_fuse(prob, feat)\n", + " loss = F.binary_cross_entropy(prob, y.astype('float32'))\n", + " ys.append(y.numpy()); ps.append(prob.numpy())\n", + " total_loss += float(loss); total_batches += 1\n", + " y_true = np.concatenate(ys, 0); y_prob = np.concatenate(ps, 0)\n", + " y_pred = (y_prob >= threshold).astype(np.float32)\n", + " per_f1, macro_f1, micro_f1 = f1_per_class(y_true, y_pred)\n", + " ap_micro = average_precision_micro(y_true, y_prob)\n", + " return {\"loss\": total_loss/max(1,total_batches), \"macro_f1\": macro_f1, \"micro_f1\": micro_f1,\n", + " \"per_class_f1\": per_f1.tolist(), \"micro_AP\": ap_micro}\n", + "\n", + "# ====================== 检索增强（cos / l2；k 邻居软加权；概率融合） ======================\n", + "class Retriever:\n", + " def __init__(self, sim_metric: str = 'cos', k: int = 8, alpha: float = 0.3, tau: float = 0.5):\n", + " assert sim_metric in ['cos', 'l2']\n", + " self.sim_metric = sim_metric; self.k = k; self.alpha = alpha; self.tau = tau\n", + " self.keys = None; self.labels = None\n", + " @paddle.no_grad()\n", + " def build(self, model: nn.Layer, loader: DataLoader):\n", + " model.eval()\n", + " feats, labs = [], []\n", + " for x_vid, x_vec, y in loader:\n", + " f = model.encode(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " feats.append(f.numpy()); labs.append(y.numpy())\n", + " self.keys = paddle.to_tensor(np.concatenate(feats, 0)).astype('float32')\n", + " self.labels = paddle.to_tensor(np.concatenate(labs, 0)).astype('float32')\n", + " self.keys_norm = F.normalize(self.keys, axis=-1)\n", + " @paddle.no_grad()\n", + " def query_and_fuse(self, model_probs: paddle.Tensor, test_feat: paddle.Tensor) -> paddle.Tensor:\n", + " B, D = test_feat.shape\n", + " if self.sim_metric == 'cos':\n", + " q = F.normalize(test_feat, axis=-1)\n", + " sim = paddle.matmul(q, self.keys_norm, transpose_y=True)\n", + " w = F.softmax(sim / self.tau, axis=-1)\n", + " else:\n", + " q2 = paddle.sum(test_feat * test_feat, axis=-1, keepdim=True)\n", + " k2 = paddle.sum(self.keys * self.keys, axis=-1, keepdim=True).transpose([1,0])\n", + " dot = paddle.matmul(test_feat, self.keys, transpose_y=True)\n", + " dist2 = q2 + k2 - 2.0 * dot\n", + " w = F.softmax(-dist2 / self.tau, axis=-1)\n", + " topk_val, topk_idx = paddle.topk(w, k=min(self.k, w.shape[1]), axis=-1)\n", + " picked_labels = paddle.gather(self.labels, topk_idx.reshape([-1]), axis=0)\n", + " C = self.labels.shape[1]\n", + " picked_labels = picked_labels.reshape([B, -1, C])\n", + " w_norm = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9)\n", + " p_knn = paddle.sum(picked_labels * w_norm.unsqueeze(-1), axis=1)\n", + " p_final = (1.0 - self.alpha) * model_probs + self.alpha * p_knn\n", + " return p_final.clip(1e-6, 1-1e-6)\n", + "\n", + "# ====================== ToyDataset（T=365, N=24） ======================\n", + "class ToyTwoModalDataset(Dataset):\n", + " def __init__(self, n: int, seed: int = 0, T: int = 365, C: int = 20, H: int = 20, W: int = 20, N: int = 24):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.n = n; self.T=T; self.C=C; self.H=H; self.W=W; self.N=N\n", + " self.video = rng.normal(size=(n, T, C, H, W, N)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + " vid_hwn = self.video.mean(axis=(3,4,5)) # (n,T,C)\n", + " vid_avg = vid_hwn.mean(axis=1) # (n,C)\n", + " Wv = rng.normal(size=(C,4)); Wt = rng.normal(size=(424,4))\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n,4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + " def __len__(self): return self.n\n", + "\n", + "# ====================== 训练入口 ======================\n", + "if __name__ == \"__main__\":\n", + " paddle.seed(2025)\n", + " # 数据\n", + " T, C, H, W, N = 365, 20, 20, 20, 24\n", + " train_ds = ToyTwoModalDataset(n=32, seed=42, T=T, C=C, H=H, W=W, N=N)\n", + " val_ds = ToyTwoModalDataset(n=16, seed=233, T=T, C=C, H=H, W=W, N=N)\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)),\n", + " paddle.to_tensor(np.stack(vecs, 0)),\n", + " paddle.to_tensor(np.stack(ys, 0)))\n", + " train_loader = DataLoader(train_ds, batch_size=1, shuffle=True, drop_last=False, collate_fn=collate_fn)\n", + " val_loader = DataLoader(val_ds, batch_size=1, shuffle=False, drop_last=False, collate_fn=collate_fn)\n", + "\n", + " # 类别不平衡权重（可选）\n", + " y_train = np.stack([y for _, _, y in train_ds], 0)\n", + " pos_ratio = np.clip(y_train.mean(axis=0), 1e-3, 1-1e-3)\n", + " pos_weight = paddle.to_tensor(((1-pos_ratio)/pos_ratio).astype('float32'))\n", + "\n", + " # 模型：Self-Attn + AFNO 两路，并行 + MoE FFN\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=C, vid_h=H, vid_w=W, vid_frames=T, depth_n=N,\n", + " vec_dim=424,\n", + " d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " moe_temporal_attn=True, moe_temporal_afno=True,\n", + " moe_fused=False, moe_tabm=False,\n", + " afno_modes=32\n", + " )\n", + " optimizer = paddle.optimizer.Adam(learning_rate=3e-4, parameters=model.parameters())\n", + "\n", + " # 训练（演示用：小 epoch）\n", + " best_macro_f1, best = -1.0, None\n", + " for ep in range(1, 2+1):\n", + " model.train(); total_loss, total_batches = 0.0, 0\n", + " for x_vid, x_vec, y in train_loader:\n", + " logits, _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " cls = F.binary_cross_entropy_with_logits(logits, y.astype('float32'), pos_weight=pos_weight)\n", + " cls.backward()\n", + " optimizer.step(); optimizer.clear_grad()\n", + " total_loss += float(cls); total_batches += 1\n", + " val_metrics = evaluate(model, val_loader, threshold=0.5, retriever=None)\n", + " print(f\"[Epoch {ep:02d}] train_loss={total_loss/max(1,total_batches):.4f} | \"\n", + " f\"val_loss={val_metrics['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics['micro_AP']:.4f}\")\n", + " if val_metrics[\"macro_f1\"] > best_macro_f1:\n", + " best_macro_f1 = val_metrics[\"macro_f1\"]\n", + " best = {k: v.clone() for k, v in model.state_dict().items()}\n", + " if best is not None:\n", + " model.set_state_dict(best); print(f\"Loaded best state with macro_f1={best_macro_f1:.4f}\")\n", + "\n", + " # 检索库 + 检索增强评估\n", + " retr = Retriever(sim_metric='cos', k=8, alpha=0.3, tau=0.5)\n", + " retr.build(model, DataLoader(train_ds, batch_size=1, shuffle=False, collate_fn=collate_fn))\n", + " val_metrics_knn = evaluate(model, val_loader, threshold=0.5, retriever=retr)\n", + " print(f\"[RkNN] val_loss={val_metrics_knn['loss']:.4f} | \"\n", + " f\"macro_f1={val_metrics_knn['macro_f1']:.4f} | \"\n", + " f\"micro_f1={val_metrics_knn['micro_f1']:.4f} | \"\n", + " f\"per_class_f1={val_metrics_knn['per_class_f1']} | \"\n", + " f\"micro_AP={val_metrics_knn['micro_AP']:.4f}\")\n" + ], + "metadata": { + "id": "KzXknA11mNYw" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math, os\n", + "from typing import Optional, Tuple, List\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# ============ 基本设置 ============\n", + "os.makedirs(\"viz_out\", exist_ok=True)\n", + "\n", + "# ============ 工具：正弦位置编码 ============\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 4096):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + " def forward(self, x): # (B,T,D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ============ TabM（占位，可替换为你的实现） ============\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden), nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + " def forward(self, x_num: paddle.Tensor):\n", + " return self.net(x_num)\n", + "\n", + "# ============ 3D ResNet18 ============\n", + "class BasicBlock3D(nn.Layer):\n", + " expansion = 1\n", + " def __init__(self, in_planes, planes, stride=(1,1,1), downsample=None):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv3D(in_planes, planes, 3, stride=stride, padding=1, bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(planes)\n", + " self.relu = nn.ReLU()\n", + " self.conv2 = nn.Conv3D(planes, planes, 3, stride=1, padding=1, bias_attr=False)\n", + " self.bn2 = nn.BatchNorm3D(planes)\n", + " self.downsample = downsample\n", + " def forward(self, x):\n", + " identity = x\n", + " out = self.relu(self.bn1(self.conv1(x)))\n", + " out = self.bn2(self.conv2(out))\n", + " if self.downsample is not None:\n", + " identity = self.downsample(x)\n", + " out = self.relu(out + identity)\n", + " return out\n", + "\n", + "class ResNet3D(nn.Layer):\n", + " def __init__(self, block, layers, in_channels=20, base_width=64):\n", + " super().__init__()\n", + " self.in_planes = base_width\n", + " self.conv1 = nn.Conv3D(in_channels, self.in_planes, kernel_size=(3,7,7),\n", + " stride=(1,2,2), padding=(1,3,3), bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(self.in_planes)\n", + " self.relu = nn.ReLU()\n", + " self.maxpool = nn.MaxPool3D(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))\n", + " self.layer1 = self._make_layer(block, base_width, layers[0], stride=(1,1,1))\n", + " self.layer2 = self._make_layer(block, base_width*2, layers[1], stride=(2,2,2))\n", + " self.layer3 = self._make_layer(block, base_width*4, layers[2], stride=(2,2,2))\n", + " self.layer4 = self._make_layer(block, base_width*8, layers[3], stride=(2,2,2))\n", + " self.out_dim = base_width*8 # 512\n", + " self.pool = nn.AdaptiveAvgPool3D(output_size=1)\n", + " def _make_layer(self, block, planes, blocks, stride=(1,1,1)):\n", + " downsample = None\n", + " if stride != (1,1,1) or self.in_planes != planes * block.expansion:\n", + " downsample = nn.Sequential(\n", + " nn.Conv3D(self.in_planes, planes * block.expansion, 1, stride=stride, bias_attr=False),\n", + " nn.BatchNorm3D(planes * block.expansion),\n", + " )\n", + " layers = [block(self.in_planes, planes, stride=stride, downsample=downsample)]\n", + " self.in_planes = planes * block.expansion\n", + " for _ in range(1, blocks):\n", + " layers.append(block(self.in_planes, planes))\n", + " return nn.Sequential(*layers)\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.relu(self.bn1(self.conv1(x)))\n", + " x = self.maxpool(x)\n", + " x = self.layer1(x); x = self.layer2(x); x = self.layer3(x); x = self.layer4(x)\n", + " x = self.pool(x) # (B, 512, 1,1,1)\n", + " x = paddle.flatten(x, 1) # (B, 512)\n", + " return x\n", + "\n", + "class Volume3DEncoder(nn.Layer):\n", + " \"\"\"\n", + " 附带特征/梯度捕获，用于 3D Grad-CAM：\n", + " - forward_post_hook 里：先缓存 feat\n", + " - 若 out 可梯度，则注册 backward hook；否则跳过（避免 stop_gradient 报错）\n", + " \"\"\"\n", + " def __init__(self, in_channels: int = 20, base: int = 64, dropout: float = 0.0):\n", + " super().__init__()\n", + " self.backbone = ResNet3D(BasicBlock3D, [2,2,2,2], in_channels=in_channels, base_width=base)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.out_dim = self.backbone.out_dim # 512\n", + " self._feat = None\n", + " self._grad = None\n", + "\n", + " def _save_feat_grad(layer, inp, out):\n", + " self._feat = out # (B, 512, D',H',W')\n", + " if getattr(out, \"stop_gradient\", False):\n", + " return\n", + " def _save_grad(grad):\n", + " self._grad = grad\n", + " out.register_hook(_save_grad)\n", + "\n", + " self.backbone.layer4.register_forward_post_hook(_save_feat_grad)\n", + "\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.backbone(x)\n", + " x = self.drop(x)\n", + " return x\n", + "\n", + "# ============ MoE ============\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(F.relu(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self, n_experts=8, top_k=1, d_ff=2048, dropout=0.1,\n", + " router_temp=0.5, use_gumbel=False):\n", + " self.n_experts=n_experts; self.top_k=top_k; self.d_ff=d_ff; self.dropout=dropout\n", + " self.router_temp=router_temp; self.use_gumbel=use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"缓存最近一次路由概率/索引，便于可解释与聚类\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model); self.drop = nn.Dropout(cfg.dropout)\n", + " self.last_router_probs = None\n", + " self.last_topk_idx = None\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u)); logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + " def forward(self, x):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3: B,T,D = orig_shape; X = x.reshape([B*T, D])\n", + " else: X = x\n", + " N,D = X.shape\n", + " logits = self.router(X); probs = self._router_probs(logits)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1)\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list=[]\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64')\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1)\n", + " picked_i = paddle.gather_nd(all_out, idx_nd)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + " Y = self.drop(Y); Y = self.ln(Y + X)\n", + " self.last_router_probs = probs.detach()\n", + " self.last_topk_idx = topk_idx.detach()\n", + " if len(orig_shape)==3: Y = Y.reshape([B,T,D])\n", + " return Y\n", + "\n", + "class MoEHead(nn.Layer):\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " self.last_router_probs = None\n", + " self.last_topk_idx = None\n", + " def forward(self, tok):\n", + " y = self.moe(tok.unsqueeze(1)).squeeze(1)\n", + " self.last_router_probs = self.moe.last_router_probs\n", + " self.last_topk_idx = self.moe.last_topk_idx\n", + " return y\n", + "\n", + "# ============ 原生 MHA + 手工回算注意力 ============\n", + "class _NativeMHAWithAttn(nn.Layer):\n", + " \"\"\"\n", + " 包装 nn.MultiHeadAttention：\n", + " - 正式输出用原生 MHA (性能/数值一致)\n", + " - 注意力矩阵用相同权重手工回算（兼容旧版不返回 attn 的情况）\n", + " \"\"\"\n", + " def __init__(self, d_model: int, nhead: int, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.mha = nn.MultiHeadAttention(embed_dim=d_model, num_heads=nhead, dropout=dropout)\n", + " self.nhead = nhead\n", + " assert d_model % nhead == 0\n", + " self.d_head = d_model // nhead\n", + " self.last_attn = None # (B, H, T, T)\n", + "\n", + " def forward(self, x_tb: paddle.Tensor) -> paddle.Tensor:\n", + " \"\"\"\n", + " x_tb: (T,B,D)\n", + " return: (T,B,D)\n", + " \"\"\"\n", + " # 1) 原生前向（不同版本的返回值差异：此处统一只拿输出）\n", + " out_tb = self.mha(x_tb, x_tb, x_tb)\n", + "\n", + " # 2) 手工回算注意力：用同一组投影权重\n", + " q = self.mha.q_proj(x_tb) # (T,B,D)\n", + " k = self.mha.k_proj(x_tb)\n", + " v = self.mha.v_proj(x_tb)\n", + " T, B, D = q.shape\n", + " H, Dh = self.nhead, self.d_head\n", + "\n", + " def split(tb): # (T,B,D)->(B,H,T,Dh)\n", + " tb = tb.transpose([1, 0, 2]) # (B,T,D)\n", + " return tb.reshape([B, T, H, Dh]).transpose([0, 2, 1, 3])\n", + "\n", + " qh, kh, vh = split(q), split(k), split(v) # (B,H,T,Dh)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(Dh) # (B,H,T,T)\n", + " attn = F.softmax(scores, axis=-1)\n", + " self.last_attn = attn.detach()\n", + "\n", + " return out_tb # (T,B,D)\n", + "\n", + "# ============ Self-Attention Transformer（记录注意力） ============\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None, capture_attn: bool = True):\n", + " super().__init__()\n", + " self.use_moe = use_moe; self.capture_attn = capture_attn\n", + " self.self_attn = _NativeMHAWithAttn(d_model, nhead, dropout)\n", + " self.ln1 = nn.LayerNorm(d_model); self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model))\n", + " self.do2 = nn.Dropout(dropout)\n", + " self.last_attn = None # (B,H,T,T)\n", + " def forward(self, x): # (B,T,D)\n", + " h = self.ln1(x)\n", + " h_tb = paddle.transpose(h, [1,0,2]) # (T,B,D)\n", + " out_tb = self.self_attn(h_tb) # (T,B,D)\n", + " if self.capture_attn:\n", + " self.last_attn = self.self_attn.last_attn\n", + " out = paddle.transpose(out_tb, [1,0,2]) # (B,T,D)\n", + " x = x + self.do1(out)\n", + " if self.use_moe:\n", + " x = self.moe(x)\n", + " else:\n", + " x = x + self.do2(self.ffn(x))\n", + " return x\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=4, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=4096, use_moe: bool = True, moe_cfg: MoEConfig = None, capture_attn=True):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg, capture_attn=capture_attn)\n", + " for _ in range(num_layers)\n", + " ])\n", + " self.last_attn_all_layers: List[paddle.Tensor] = []\n", + " def forward(self, x):\n", + " x = self.pos(x)\n", + " self.last_attn_all_layers = []\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " if layer.last_attn is not None:\n", + " self.last_attn_all_layers.append(layer.last_attn) # (B,H,T,T)\n", + " return x\n", + "\n", + "# ============ AFNO(1D) + MoE FFN ============\n", + "class AFNO1DLayer(nn.Layer):\n", + " def __init__(self, d_model: int, modes: int = 32, num_blocks: int = 8, shrink: float = 0.01, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % num_blocks == 0\n", + " self.d_model=d_model; self.modes=modes; self.num_blocks=num_blocks; self.block=d_model//num_blocks\n", + " scale=1.0/math.sqrt(self.block); init = nn.initializer.Uniform(-scale, scale)\n", + " self.w1r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w1i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w2r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w2i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.ln = nn.LayerNorm(d_model); self.drop = nn.Dropout(dropout); self.shrink = shrink\n", + " def _cl(self, xr, xi, Wr, Wi):\n", + " out_r = paddle.matmul(xr, Wr) - paddle.matmul(xi, Wi)\n", + " out_i = paddle.matmul(xr, Wi) + paddle.matmul(xi, Wr)\n", + " return out_r, out_i\n", + " def forward(self, x): # (B,T,D)\n", + " B,T,D = x.shape; Kmax=T//2+1; K=min(self.modes, Kmax)\n", + " h=self.ln(x); h_td=h.transpose([0,2,1]); h_ft=paddle.fft.rfft(h_td) # (B,D,F)\n", + " h_ft=h_ft.reshape([B, self.num_blocks, self.block, Kmax])\n", + " xk=h_ft[:,:,:, :K].transpose([0,1,3,2]) # (B,G,K,Cb)\n", + " xr, xi = paddle.real(xk), paddle.imag(xk)\n", + " yr, yi = self._cl(xr, xi, self.w1r, self.w1i)\n", + " yr = F.gelu(yr); yi = F.gelu(yi)\n", + " yr = F.softshrink(yr, threshold=self.shrink); yi = F.softshrink(yi, threshold=self.shrink)\n", + " yr, yi = self._cl(yr, yi, self.w2r, self.w2i)\n", + " yk = paddle.complex(yr, yi).transpose([0,1,3,2]).reshape([B,D,K])\n", + " out_ft = paddle.zeros([B,D,Kmax], dtype='complex64')\n", + " out_ft[:,:, :K] = yk\n", + " out_td = paddle.fft.irfft(out_ft, n=T)\n", + " out = out_td.transpose([0,2,1])\n", + " out = self.drop(out)\n", + " return x + out\n", + "\n", + "class AFNOTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, num_layers=2, modes=32, dropout=0.1,\n", + " d_ff=1024, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.layers = nn.LayerList([AFNO1DLayer(d_model, modes, 8, 0.01, dropout) for _ in range(num_layers)])\n", + " self.use_moe = use_moe\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model))\n", + " self.do = nn.Dropout(dropout)\n", + " def forward(self, x):\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " if self.use_moe:\n", + " x = self.moe(x)\n", + " else:\n", + " x = x + self.do(self.ffn(x))\n", + " return x\n", + "\n", + "# ============ Cross-Attention（记录注意力） ============\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_head = d_model // nhead; self.nhead = nhead\n", + " self.Wq = nn.Linear(d_model, d_model); self.Wk = nn.Linear(d_model, d_model); self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model); self.drop = nn.Dropout(dropout); self.ln = nn.LayerNorm(d_model)\n", + " self.last_attn = None # (B, H, Nq, Nk)\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape; Nk = kv.shape[1]\n", + " def split(t): return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0,2,1,3])\n", + " qh = split(self.Wq(q)); kh = split(self.Wk(kv)); vh = split(self.Wv(kv))\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head) # (B,H,Nq,Nk)\n", + " attn = F.softmax(scores, axis=-1)\n", + " self.last_attn = attn.detach()\n", + " ctx = paddle.matmul(attn, vh).transpose([0,2,1,3]).reshape([B,Nq,D])\n", + " out = self.drop(self.proj(ctx))\n", + " return self.ln(out + q)\n", + "\n", + "class BiModalCrossFusion(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(nn.Linear(2*d_model, fuse_hidden), nn.ReLU(), nn.Dropout(dropout))\n", + " self.out_dim = fuse_hidden\n", + " self.last_attn_v_from_t = None # (B,H,1,1)\n", + " self.last_attn_t_from_v = None # (B,H,1,T)\n", + " def forward(self, video_seq, tabm_tok):\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq)\n", + " self.last_attn_v_from_t = self.ca_v_from_t.last_attn\n", + " self.last_attn_t_from_v = self.ca_t_from_v.last_attn\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1).squeeze(1)\n", + " return self.fuse(fused)\n", + "\n", + "# ============ 总模型 ============\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self, vid_channels=20, vid_frames=365, depth_n=24,\n", + " vec_dim=424, d_model=512, nhead=4, n_trans_layers=2, trans_ff=1024,\n", + " tabm_hidden=512, dropout=0.1, num_labels=4,\n", + " moe_temporal_attn=True, moe_temporal_afno=True, moe_fused=False, moe_tabm=False,\n", + " afno_modes=32):\n", + " super().__init__()\n", + " self.vol_encoder = Volume3DEncoder(in_channels=vid_channels, dropout=dropout)\n", + " self.trans_attn = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers, d_ff=trans_ff, dropout=dropout,\n", + " max_len=vid_frames, use_moe=moe_temporal_attn, moe_cfg=MoEConfig(d_ff=max(2048,trans_ff), n_experts=8),\n", + " capture_attn=True\n", + " )\n", + " self.trans_afno = AFNOTransformerFlexible(\n", + " d_model=d_model, num_layers=n_trans_layers, modes=afno_modes, dropout=dropout,\n", + " d_ff=trans_ff, use_moe=moe_temporal_afno, moe_cfg=MoEConfig(d_ff=max(2048,trans_ff), n_experts=8)\n", + " )\n", + " self.video_merge = nn.Linear(2*d_model, d_model)\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=MoEConfig(d_ff=1024, n_experts=6))\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=MoEConfig(d_ff=1024, n_experts=6))\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + " self.depth_n = depth_n\n", + " def encode(self, x_video, x_vec):\n", + " B,T,C,H,W,N = x_video.shape\n", + " assert N == self.depth_n\n", + " xvt = x_video.transpose([0,1,2,5,3,4]).reshape([B*T, C, N, H, W])\n", + " f_frame = self.vol_encoder(xvt) # (B*T,512)\n", + " seq = f_frame.reshape([B, T, -1]) # (B,T,512)\n", + " z_attn = self.trans_attn(seq)\n", + " z_afno = self.trans_afno(seq)\n", + " z_vid = self.video_merge(paddle.concat([z_attn, z_afno], axis=-1))\n", + " z_tabm = self.tabm(x_vec); z_tabm = self.tabm_proj(z_tabm)\n", + " if self.moe_tabm:\n", + " z_tabm = self.tabm_moe(z_tabm)\n", + " fused = self.fusion(z_vid, z_tabm)\n", + " if self.moe_fused:\n", + " fused = self.fused_moe(fused)\n", + " return fused\n", + " def forward(self, x_video, x_vec):\n", + " fused = self.encode(x_video, x_vec)\n", + " logits = self.head(fused)\n", + " return logits\n", + "\n", + "# ============ 3D Grad-CAM ============\n", + "class GradCAM3D:\n", + " \"\"\"\n", + " CAM = ReLU( sum_c( w_c * A_c ) ), w_c = GAP(grad_c)\n", + " 输出 (N,H,W)，若无 scipy 则返回特征尺度 (D',H',W')\n", + " \"\"\"\n", + " def __init__(self, model: TwoModalMultiLabelModel):\n", + " self.model = model\n", + " @paddle.no_grad()\n", + " def _trilinear_upsample(self, vol, out_shape):\n", + " try:\n", + " from scipy.ndimage import zoom\n", + " Dz = out_shape[0] / vol.shape[0]\n", + " Dy = out_shape[1] / vol.shape[1]\n", + " Dx = out_shape[2] / vol.shape[2]\n", + " return zoom(vol, (Dz, Dy, Dx), order=1)\n", + " except Exception:\n", + " return vol\n", + " def generate(self, x_video, x_vec, target_class: int = 0, time_index: int = 0):\n", + " assert x_video.shape[0] == 1, \"Grad-CAM 演示请用单样本 B=1\"\n", + " self.model.eval()\n", + " B,T,C,H,W,N = x_video.shape\n", + " self.model.clear_gradients()\n", + " logits = self.model(x_video.astype('float32'), x_vec.astype('float32')) # (1,num_labels)\n", + " cls = logits[0, target_class]\n", + " cls.backward()\n", + " feat = self.model.vol_encoder._feat # (1,512,D',H',W')\n", + " grad = self.model.vol_encoder._grad\n", + " assert (feat is not None) and (grad is not None), \"未捕获到特征/梯度（检查 hook & 是否有梯度前向）\"\n", + " feat_np = feat.numpy()[0]; grad_np = grad.numpy()[0]\n", + " w = grad_np.mean(axis=(1,2,3)) # (512,)\n", + " cam = np.maximum(0, np.tensordot(w, feat_np, axes=(0,0))) # (D',H',W')\n", + " cam = cam - cam.min(); cam = cam / (cam.max() + 1e-8)\n", + " cam_up = self._trilinear_upsample(cam, (N, H, W))\n", + " return cam_up # (N,H,W) or (D',H',W')\n", + "\n", + "# ============ MoE 路由聚类工具 ============\n", + "def kmeans_numpy(X: np.ndarray, K: int = 4, iters: int = 50, seed: int = 0):\n", + " rng = np.random.default_rng(seed)\n", + " N,D = X.shape\n", + " cent = X[rng.choice(N, K, replace=False)]\n", + " for _ in range(iters):\n", + " dist2 = ((X[:,None,:]-cent[None,:,:])**2).sum(axis=2) # (N,K)\n", + " idx = dist2.argmin(axis=1)\n", + " new_cent = np.stack([X[idx==k].mean(axis=0) if np.any(idx==k) else cent[k] for k in range(K)], 0)\n", + " if np.allclose(new_cent, cent): break\n", + " cent = new_cent\n", + " return idx, cent\n", + "\n", + "def collect_moe_routing_vectors(model: TwoModalMultiLabelModel, loader: DataLoader,\n", + " branch: str = \"temporal_attn\", topk_hist: bool = True):\n", + " model.eval()\n", + " vecs = []\n", + " for x_vid, x_vec, y in loader:\n", + " _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if branch == \"temporal_attn\":\n", + " moe = None\n", + " for lyr in model.trans_attn.layers[::-1]:\n", + " if hasattr(lyr, \"moe\"):\n", + " moe = lyr.moe; break\n", + " elif branch == \"temporal_afno\":\n", + " moe = model.trans_afno.moe if hasattr(model.trans_afno, \"moe\") else None\n", + " elif branch == \"tabm\":\n", + " moe = model.tabm_moe.moe if getattr(model, \"moe_tabm\", False) else None\n", + " else:\n", + " moe = model.fused_moe.moe if getattr(model, \"moe_fused\", False) else None\n", + " if moe is None or moe.last_router_probs is None:\n", + " continue\n", + " probs = moe.last_router_probs.numpy() # (N_tokens, E)\n", + " if topk_hist:\n", + " top1 = moe.last_topk_idx.numpy()[:,0] # (N_tokens,)\n", + " E = probs.shape[1]\n", + " hist = np.bincount(top1, minlength=E).astype(\"float32\")\n", + " hist = hist / (hist.sum() + 1e-9)\n", + " vecs.append(hist)\n", + " else:\n", + " vecs.append(probs.mean(axis=0))\n", + " return np.stack(vecs, 0) if len(vecs)>0 else None\n", + "\n", + "# ============ Toy 数据集 ============\n", + "class ToyTwoModalDataset(Dataset):\n", + " \"\"\"\n", + " 返回：\n", + " x_video: (T=365, C=20, H=20, W=20, N=24)\n", + " x_vec: (424,)\n", + " y: (4,) 0/1\n", + " \"\"\"\n", + " def __init__(self, n: int, seed: int = 0, T: int = 365, C: int = 20, H: int = 20, W: int = 20, N: int = 24):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.video = rng.normal(size=(n, T, C, H, W, N)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + " # 造标签：体素均值 → (n,C) → 线性到 4 类\n", + " vid_hwn = self.video.mean(axis=(3,4,5)) # (n,T,C)\n", + " vid_avg = vid_hwn.mean(axis=1) # (n,C)\n", + " Wv = rng.normal(size=(C,4)); Wt = rng.normal(size=(424,4))\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n,4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + " def __len__(self): return len(self.y)\n", + "\n", + "# ============ 小工具：绘图 ============\n", + "def show_heatmap_2d(arr2d: np.ndarray, title: str, save_path: Optional[str] = None):\n", + " plt.figure(); plt.imshow(arr2d, interpolation='nearest'); plt.title(title); plt.colorbar()\n", + " if save_path: plt.savefig(save_path, bbox_inches='tight');\n", + " plt.show(); plt.close()\n", + "\n", + "def show_attention_matrix(attn: np.ndarray, title: str, save_path: Optional[str] = None):\n", + " # attn: (B,H,T,T) 或 (B,H,1,T) 或 (B,H,1,1)\n", + " if attn.ndim == 4 and attn.shape[2] == 1 and attn.shape[3] == 1:\n", + " attn = attn[0,:,0,0][:,None] # (H,1)\n", + " elif attn.ndim == 4 and attn.shape[2] == 1:\n", + " attn = attn[0] # (H,1,T)\n", + " elif attn.ndim == 4:\n", + " attn = attn[0] # (H,T,T)\n", + " plt.figure(figsize=(5,4))\n", + " if attn.ndim == 2: # (H,1)\n", + " plt.imshow(attn, aspect='auto', interpolation='nearest')\n", + " elif attn.ndim == 3: # 多头\n", + " H = attn.shape[0]\n", + " cols = int(np.ceil(np.sqrt(H))); rows = int(np.ceil(H/cols))\n", + " fig, axes = plt.subplots(rows, cols, figsize=(3*cols, 3*rows))\n", + " axes = axes.flatten()\n", + " for h in range(H):\n", + " axes[h].imshow(attn[h], interpolation='nearest'); axes[h].set_title(f\"head {h}\")\n", + " for k in range(H, len(axes)): axes[k].axis('off')\n", + " fig.suptitle(title)\n", + " if save_path: fig.savefig(save_path, bbox_inches='tight')\n", + " plt.show(); plt.close(fig); return\n", + " plt.title(title); plt.colorbar()\n", + " if save_path: plt.savefig(save_path, bbox_inches='tight')\n", + " plt.show(); plt.close()\n", + "\n", + "# ============ Demo：可解释可视化 ============\n", + "if __name__ == \"__main__\":\n", + " # 1) 构造“已训练好”的模型（这里随机权重示意）\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=20, vid_frames=365, depth_n=24,\n", + " vec_dim=424, d_model=512, nhead=4, n_trans_layers=2, trans_ff=512,\n", + " tabm_hidden=256, dropout=0.1, num_labels=4,\n", + " moe_temporal_attn=True, moe_temporal_afno=True,\n", + " moe_fused=False, moe_tabm=False, afno_modes=32\n", + " )\n", + " model.eval()\n", + "\n", + " # 2) 取一个样本\n", + " toy = ToyTwoModalDataset(n=8, seed=123, T=365, C=20, H=20, W=20, N=24)\n", + " x_video, x_vec, y = toy[0]\n", + " x_video = paddle.to_tensor(x_video[None, ...]) # (1,T,C,H,W,N)\n", + " x_vec = paddle.to_tensor(x_vec[None, ...]) # (1,424)\n", + "\n", + " # 3) 3D Grad-CAM：一次“有梯度”的前向 + 反传\n", + " model.clear_gradients()\n", + " logits = model(x_video.astype('float32'), x_vec.astype('float32'))\n", + " target_class = int(paddle.argmax(logits, axis=-1)[0])\n", + " cam3d = GradCAM3D(model).generate(\n", + " x_video.astype('float32'), x_vec.astype('float32'),\n", + " target_class=target_class, time_index=0\n", + " ) # (N,H,W) or (D',H',W')\n", + "\n", + " # 展示几个深度切片\n", + " Nz = cam3d.shape[0]\n", + " for z in [0, Nz//3, 2*Nz//3, Nz-1]:\n", + " show_heatmap_2d(cam3d[z], f\"Grad-CAM depth={z}\", save_path=f\"viz_out/gradcam_z{z}.png\")\n", + "\n", + " # 4) Self-Attention & Cross-Attention 注意力矩阵\n", + " with paddle.no_grad():\n", + " _ = model.encode(x_video.astype('float32'), x_vec.astype('float32'))\n", + " # Self-Attn（最后一层）\n", + " last_attn_list = model.trans_attn.last_attn_all_layers\n", + " if len(last_attn_list) > 0:\n", + " attn = last_attn_list[-1].numpy() # (B,H,T,T)\n", + " attn_crop = attn[:, :, :64, :64]\n", + " show_attention_matrix(attn_crop, \"Self-Attention (last layer, first 64 tokens)\",\n", + " save_path=\"viz_out/self_attn_lastlayer_64.png\")\n", + " print(\"Self-Attn matrix shape:\", attn.shape)\n", + " else:\n", + " print(\"Self-Attn not captured.\")\n", + " # Cross-Attn\n", + " if model.fusion.last_attn_v_from_t is not None:\n", + " show_attention_matrix(model.fusion.last_attn_v_from_t.numpy(),\n", + " \"Cross-Attn v<-t (token→token)\",\n", + " save_path=\"viz_out/cross_attn_v_from_t.png\")\n", + " if model.fusion.last_attn_t_from_v is not None:\n", + " attn_tv = model.fusion.last_attn_t_from_v.numpy()\n", + " attn_tv_crop = attn_tv[:,:,:, :64]\n", + " show_attention_matrix(attn_tv_crop,\n", + " \"Cross-Attn t<-v (token←video_seq first 64)\",\n", + " save_path=\"viz_out/cross_attn_t_from_v_64.png\")\n", + "\n", + " # 5) MoE 路由聚类（示例用 toy 数据）\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)),\n", + " paddle.to_tensor(np.stack(vecs, 0)),\n", + " paddle.to_tensor(np.stack(ys, 0)))\n", + " train_loader = DataLoader(toy, batch_size=1, shuffle=False, collate_fn=collate_fn)\n", + " moe_vecs = collect_moe_routing_vectors(model, train_loader, branch=\"temporal_attn\", topk_hist=True)\n", + " if moe_vecs is not None:\n", + " idx, cent = kmeans_numpy(moe_vecs, K=4, iters=100, seed=0)\n", + " print(\"\\n[MoE Routing Clusters @ temporal_attn]\")\n", + " for k in range(4):\n", + " sel = (idx==k)\n", + " if np.any(sel):\n", + " mean_vec = moe_vecs[sel].mean(axis=0)\n", + " dom = int(mean_vec.argmax())\n", + " print(f\" - Cluster {k}: size={int(sel.sum())}, dominant_expert={dom}, mean_dist={np.round(mean_vec,3)}\")\n", + " # 保存热图\n", + " plt.figure(figsize=(6,4))\n", + " plt.imshow(moe_vecs, aspect='auto', interpolation='nearest')\n", + " plt.title(\"Samples × Experts (routing histogram)\"); plt.xlabel(\"Expert\"); plt.ylabel(\"Sample\")\n", + " plt.colorbar(); plt.savefig(\"viz_out/moe_routing_heatmap.png\", bbox_inches='tight')\n", + " plt.show(); plt.close()\n", + " else:\n", + " print(\"MoE routing not available on selected branch.\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "j6NZgsBmZk7u", + "outputId": "2834b1eb-d29b-4095-ccb8-35e72eba8ec5" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAGzCAYAAABO7D91AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAATxpJREFUeJzt3X1cVGX+P/7XoDIDKpiC3Cgo4g0qCopJWCkmiegaKqmZuyIori70WWVNo0zI6kd33qUmtSlUSqar4pp+cYEEc0XNGzKzSAkBFTDdAEG5iTm/P1rOemQG5jgzIJ7Xcx/X4+Gcc13Xec8Jl7fXdZ3rqARBEEBERESkg0VrB0BEREQPLiYKREREpBcTBSIiItKLiQIRERHpxUSBiIiI9GKiQERERHoxUSAiIiK9mCgQERGRXkwUiIiISC8mCqRoc+fORe/evVs7jBbh7+8Pf3//Vru+SqVCVFRUq12fiO4PEwVqFfn5+YiKikL//v1hbW0Na2trDBo0CJGRkTh37lxrh6dXRUUFXnvtNXh5eaFTp06wsrKCp6cnli9fjmvXrulsM2PGDKhUKixfvlzn+czMTKhUKqhUKmzbtk1nnccffxwqlQqenp4m+y7mcOzYMcTFxaGsrKxVrl9WVoYFCxbA3t4eHTt2xNixY3HmzJlWiYXoYcFEgVrcl19+CU9PT3z22WcICAjA2rVrsX79egQFBeHgwYPw9vZGQUFBa4fZyM8//wxvb2+8/vrrGDRoEN5++228//77GDt2LLZs2aLzX+sVFRXYv38/evfujc8//xxNvVpFo9EgOTm50fHLly/j2LFj0Gg0pvw6ZnHs2DG89tprrZIoaLVaTJo0CcnJyYiKisI777yD69evw9/fHxcvXmzxeIgeFu1bOwBSlry8PDz33HPo1asXMjIy4OTkJDn/9ttv44MPPoCFRdM5bFVVFTp27GjOUCV+++03TJs2DaWlpcjMzMQTTzwhOf/mm2/i7bffbtRu9+7dqK+vx9atW/HUU0/hyJEjGDNmjM5rTJw4Ef/85z9x48YN2NnZiceTk5Ph4OCAfv364ddffzXtF3uI/OMf/8CxY8ewa9cuPPvsswB+H83p378/YmNjdSZhRNQ8jihQi3rnnXdQVVWFxMTERkkCALRv3x7/93//BxcXF/HY3Llz0alTJ+Tl5WHixIno3LkzZs+eDQD4+uuvMX36dLi6ukKtVsPFxQVLlizBnTt3GvWdkpICT09PaDQaeHp6Yu/evQbHvXv3bnz77bd45ZVXGiUJAGBjY4M333yz0fHt27fj6aefxtixYzFw4EBs375d7zWCg4OhVquxa9cuyfHk5GTMmDED7dq1Mzjejz76CO7u7rCyssLIkSPx9ddf66xXU1OD2NhY9O3bV7x/y5YtQ01NjaRew/qC7du3Y8CAAdBoNPDx8cGRI0fEOnFxcXjxxRcBAG5ubuJ0yuXLlyV9Nfx3UKvVGDx4MFJTUw3+Xk35xz/+AQcHB0ybNk08Zm9vjxkzZmDfvn2NvhMRGYYjCtSivvzyS/Tt2xe+vr6y2v32228IDAzEE088gffeew/W1tYAgF27duH27dtYtGgRunXrhpMnT2LDhg24cuWK5Bfuv/71L4SEhGDQoEGIj4/HzZs3ERYWhp49exp0/X/+858AgD/96U8Gx3zt2jUcPnwYn3zyCQBg1qxZWLt2LTZu3AhLS8tG9a2trREcHIzPP/8cixYtAgB8++23+P777/Hxxx8bvHZjy5Yt+POf/4xRo0Zh8eLF+Pnnn/HMM8+ga9eukgRMq9XimWeewdGjR7FgwQIMHDgQ3333HdauXYuffvoJKSkpkn6zsrLwxRdf4P/+7/+gVqvxwQcfYMKECTh58iQ8PT0xbdo0/PTTT/j888+xdu1acVTE3t5e7OPo0aPYs2cP/vKXv6Bz5854//33ERISgsLCQnTr1g0AUFdXh/LycoO+a9euXcXRp7Nnz2L48OGNRqNGjhyJjz76CD/99BOGDBliUL9EdBeBqIWUl5cLAIQpU6Y0Ovfrr78Kv/zyi1hu374tngsNDRUACC+99FKjdnfXaxAfHy+oVCqhoKBAPObt7S04OTkJZWVl4rF//etfAgChV69ezcY+bNgwwdbWttl6d3vvvfcEKysroaKiQhAEQfjpp58EAMLevXsl9Q4fPiwAEHbt2iV8+eWXgkqlEgoLCwVBEIQXX3xR6NOnjyAIgjBmzBhh8ODBTV6ztrZW6N69u+Dt7S3U1NSIxz/66CMBgDBmzBjx2GeffSZYWFgIX3/9taSPhIQEAYDw73//WzwGQAAgnDp1SjxWUFAgaDQaYerUqeKxd999VwAg5OfnN4oNgGBpaSlcunRJPPbtt98KAIQNGzY0uh+GlLuv07FjRyE8PLzRdQ8cOCAAEFJTU5u8d0SkG0cUqMVUVFQAADp16tTonL+/P7799lvx87vvvoulS5dK6jT8K/tuVlZW4p+rqqpw584djBo1CoIg4OzZs3B1dUVxcTFycnLw0ksvwdbWVqz/9NNPY9CgQaiqqjIo9s6dOzf/Je+yfft2TJo0SWzXr18/+Pj4YPv27ZgyZYrONuPHj0fXrl2xY8cOLF26FDt27MCcOXMMvuapU6dw/fp1rFq1SjJqMXfuXHFaoMGuXbswcOBAeHh44MaNG+Lxp556CgBw+PBhjBo1Sjzu5+cHHx8f8bOrqyuCg4Oxf/9+1NfXGzQ1EhAQAHd3d/Hz0KFDYWNjg59//lk85uXlhbS0NIO+r6Ojo/jnO3fuQK1WN6rTsAhU13QUETWPiQK1mIZfmJWVlY3Offjhh7h16xZKS0vxxz/+sdH59u3b65wmKCwsxMqVK/HPf/6z0UK/huHrhico+vXr16j9gAEDJI/P/fLLL6ivrxc/d+rUCZ06dWr0y6w5P/zwA86ePYs5c+bg0qVL4nF/f39s2rQJFRUVsLGxadSuQ4cOmD59OpKTkzFy5EgUFRXh+eefN/i6+r5rhw4d0KdPH8mxixcv4ocffpBMDdzt+vXrks+67l///v1x+/Zt/PLLL5Jf2vq4uro2OvbII49I/ts98sgjCAgIaLave1lZWelch1BdXS2eJyL5mChQi7G1tYWTkxPOnz/f6FzDmoV7F741UKvVjeae6+vr8fTTT+M///kPli9fDg8PD3Ts2BFXr17F3LlzodVqZcf46KOPSh7NjI2NRVxcHDw8PHD27FkUFRVJ5vn1adgPYcmSJViyZEmj87t370ZYWJjOts8//zwSEhIQFxcHLy8vDBo0SPb3MIRWq8WQIUOwZs0anecN+Z5y6Rt1EO56bLS2thb/+c9/DOrP3t5e7NPJyQnFxcWN6jQcc3Z2lhsuEYGJArWwSZMm4eOPP8bJkycxcuRIo/r67rvv8NNPP+GTTz6RDM/fO2zdq1cvAND5LH1ubq7k8/bt2yVD1A3/Cp88eTI+//xzbNu2DTExMU3GJQgCkpOTMXbsWPzlL39pdP7111/H9u3b9SYKTzzxBFxdXZGZmanzkcum3P1dG6YQgN8XCObn58PLy0s85u7ujm+//Rbjxo2DSqVqtm9d9++nn36CtbW1OCphSD/NOXbsGMaOHWtQ3fz8fHFnTW9vb3z99dfQarWSpPLEiROwtrZG//79jY6NSImYKFCLWrZsGZKTkxEeHo6MjAw4ODhIzgtNbEh0r4Z/Sd7dRhAErF+/XlLPyckJ3t7e+OSTTyTrFNLS0nDhwgXxlyvw+w6Iujz77LOIj4/Hm2++CX9/f/j5+UnO37p1C2+99RbefPNN/Pvf/8bly5exatUq8Xn+u/3000949dVXce3aNZ3/ylWpVHj//fdx9uxZWU9ZAMCIESNgb2+PhIQEhIWFiesUkpKSGm2CNGPGDBw8eBB///vfsWDBAsm5O3fuQKvVSvaqyM7OxpkzZzB8+HAAQFFREfbt24cJEyaI/y0a6huz4dL9rlF49tln8Y9//AN79uwR7/uNGzewa9cuTJ48Wef6BSJqHhMFalH9+vVDcnIyZs2ahQEDBmD27Nnw8vKCIAjIz89HcnIyLCwsDHps0cPDA+7u7li6dCmuXr0KGxsb7N69W+emRPHx8Zg0aRKeeOIJhIeH4z//+Q82bNiAwYMH61wzca8OHTpgz549CAgIwOjRozFjxgw8/vjj6NChA77//nskJyfjkUcewZtvvont27ejXbt2mDRpks6+nnnmGbzyyivYsWMHoqOjddYJDg5GcHBws3HpivONN97An//8Zzz11FOYOXMm8vPzkZiY2GiNwp/+9Cfs3LkTCxcuxOHDh/H444+jvr4eP/74I3bu3IlDhw5hxIgRYn1PT08EBgZKHo8EgNdee02s07DY8ZVXXsFzzz2HDh06YPLkybI2x7rfNQrPPvssHnvsMYSFheHChQuws7PDBx98gPr6ekmMRCRTKz5xQQp26dIlYdGiRULfvn0FjUYjWFlZCR4eHsLChQuFnJwcSd3Q0FChY8eOOvu5cOGCEBAQIHTq1Emws7MTIiIixEfuEhMTJXV3794tDBw4UFCr1cKgQYOEPXv2CKGhoQY9Htng119/FVauXCkMGTJEsLa2FjQajeDp6SnExMQIxcXFQm1trdCtWzfhySefbLIfNzc3YdiwYYIgSB+PbIohj0c2+OCDDwQ3NzdBrVYLI0aMEI4cOSKMGTNG8nikIPz+OOXbb78tDB48WFCr1cIjjzwi+Pj4CK+99ppQXl4u1gMgREZGCtu2bRP69esnqNVqYdiwYcLhw4cbXfv1118XevToIVhYWEgeYWzo4169evUSQkNDDfpezfnPf/4jzJs3T+jWrZtgbW0tjBkzRvjmm29M0jeRUqkEQcZYLxEpkkqlQmRkJDZu3NjaoRBRC+MWzkRERKQXEwUiIiLSi4kCERER6cWnHoioWVzKRKRcHFEgIiIivZgoEBERkV4PxdSDVqvFtWvX0LlzZ5NsIUtERC1LEATcunULzs7Ojd7rYkrV1dWora01uh9LS0vxzaQPu4ciUbh27ZpZXmBDREQtq6ioyKCdWe9HdXU13Hp1Qsn1+uYrN8PR0RH5+fmKSBYeikSh4fXFPeNWwEIB/9GIiB422upqXIl7Q/z/c3Oora1FyfV65J/uBZvO9z9qUXFLCzefAtTW1jJRaCsaphssNBomCkREbVhLTB/bdLYwKlFQmociUSAiIjJUvaBFvRFP/NYLWtMF0wYwUSAiIkXRQoAW958pGNO2LTLb2MumTZvQu3dvaDQa+Pr64uTJk03W37VrFzw8PKDRaDBkyBAcPHjQXKEREZGCaU3wPyUxS6LwxRdfIDo6GrGxsThz5gy8vLwQGBiI69ev66x/7NgxzJo1C/PmzcPZs2cxZcoUTJkyBefPnzdHeERERGQgsyQKa9asQUREBMLCwjBo0CAkJCTA2toaW7du1Vl//fr1mDBhAl588UUMHDgQr7/+OoYPH85X2hIRkcnVC4LRRUlMnijU1tbi9OnTCAgI+N9FLCwQEBCA7OxsnW2ys7Ml9QEgMDBQb/2amhpUVFRIChERkSEa1igYU5TE5InCjRs3UF9fDwcHB8lxBwcHlJSU6GxTUlIiq358fDxsbW3Fws2WiIiIzKNNPkgaExOD8vJysRQVFbV2SERE1EZoIaDeiKK0EQWTPx5pZ2eHdu3aobS0VHK8tLQUjo6OOts4OjrKqq9Wq6FWq00TMBERKQofj5TH5CMKlpaW8PHxQUZGhnhMq9UiIyMDfn5+Otv4+flJ6gNAWlqa3vpERETUMsyy4VJ0dDRCQ0MxYsQIjBw5EuvWrUNVVRXCwsIAAHPmzEGPHj0QHx8PAPjrX/+KMWPGYPXq1Zg0aRJ27NiBU6dO4aOPPjJHeEREpGDGPrmgtKcezJIozJw5E7/88gtWrlyJkpISeHt7IzU1VVywWFhYKHmN6KhRo5CcnIwVK1bg5ZdfRr9+/ZCSkgJPT09zhEdERAqm/W8xpr2SqASh7adGFRUVsLW1hetbb/ClUEREbZC2uhqFL61AeXk5bGxszHKNht8VP/7ggM5GvBTq1i0tPAaWmjXWBwnf9UBERIrS8PSCMe2VhIkCEREpSr0AI98eabpY2gImCkREpChcoyBPm9xwiYiIiFoGRxSIiEhRtFChHiqj2isJEwUiIlIUrfB7Maa9knDqgYiIiPTiiAIRESlKvZFTD8a0bYuYKBARkaIwUZCHUw9ERESkF0cUiIhIUbSCClrBiKcejGjbFjFRICIiReHUgzyceiAiIiK9OKJARESKUg8L1Bvx7+R6E8bSFjBRICIiRRGMXKMgcI0CERHRw4trFOThGgUiIiLSiyMKRESkKPWCBeoFI9YoKOxdD0wUiIhIUbRQQWvEgLoWysoUOPVAREREenFEgYiIFIWLGeVhokBERIpi/BoFTj0QERERAeCIAhERKczvixmNeCkUpx6IiIgeXlojt3DmUw9ERERE/8VEgYiIFKVhMaMxRa4jR45g8uTJcHZ2hkqlQkpKSrNtMjMzMXz4cKjVavTt2xdJSUmS871794ZKpWpUIiMjxTr+/v6Nzi9cuFBW7EwUiIhIUbSwMLrIVVVVBS8vL2zatMmg+vn5+Zg0aRLGjh2LnJwcLF68GPPnz8ehQ4fEOt988w2Ki4vFkpaWBgCYPn26pK+IiAhJvXfeeUdW7FyjQEREilIvqFBvxBsg76dtUFAQgoKCDK6fkJAANzc3rF69GgAwcOBAHD16FGvXrkVgYCAAwN7eXtLmrbfegru7O8aMGSM5bm1tDUdHR9kxN+CIAhER0X2oqKiQlJqaGpP1nZ2djYCAAMmxwMBAZGdn66xfW1uLbdu2ITw8HCqVNJHZvn077Ozs4OnpiZiYGNy+fVtWLBxRICIiRak38qmH+v8+9eDi4iI5Hhsbi7i4OGNCE5WUlMDBwUFyzMHBARUVFbhz5w6srKwk51JSUlBWVoa5c+dKjj///PPo1asXnJ2dce7cOSxfvhy5ubnYs2ePwbEwUSAiIkXRChbQGrEzo/a/OzMWFRXBxsZGPK5Wq42O7X5t2bIFQUFBcHZ2lhxfsGCB+OchQ4bAyckJ48aNQ15eHtzd3Q3qm4kCERHRfbCxsZEkCqbk6OiI0tJSybHS0lLY2Ng0Gk0oKChAenq6QaMEvr6+AIBLly4xUSAiItLFVFMP5uTn54eDBw9KjqWlpcHPz69R3cTERHTv3h2TJk1qtt+cnBwAgJOTk8GxMFEgIiJF0eL+nly4u71clZWVuHTpkvg5Pz8fOTk56Nq1K1xdXRETE4OrV6/i008/BQAsXLgQGzduxLJlyxAeHo6vvvoKO3fuxIEDB6SxaLVITExEaGgo2reX/krPy8tDcnIyJk6ciG7duuHcuXNYsmQJRo8ejaFDhxocu8mfeoiPj8ejjz6Kzp07o3v37pgyZQpyc3ObbJOUlNRoQwiNRmPq0IiIiFrFqVOnMGzYMAwbNgwAEB0djWHDhmHlypUAgOLiYhQWFor13dzccODAAaSlpcHLywurV6/Gxx9/LD4a2SA9PR2FhYUIDw9vdE1LS0ukp6dj/Pjx8PDwwN/+9jeEhIRg//79smI3+YhCVlYWIiMj8eijj+K3337Dyy+/jPHjx+PChQvo2LGj3nY2NjaShOLexzuIiIhM4X43Tbq7vVz+/v4Qmng99b27Lja0OXv2bJP9jh8/Xm+/Li4uyMrKkhWnLiZPFFJTUyWfk5KS0L17d5w+fRqjR4/W206lUhm1IQQREZEh7ncb5rvbK4nZv215eTkAoGvXrk3Wq6ysRK9eveDi4oLg4GB8//33euvW1NQ02uiCiIiITM+siYJWq8XixYvx+OOPw9PTU2+9AQMGYOvWrdi3bx+2bdsGrVaLUaNG4cqVKzrrx8fHw9bWViz3bnpBRESkjxYqo4uSmDVRiIyMxPnz57Fjx44m6/n5+WHOnDnw9vbGmDFjsGfPHtjb2+PDDz/UWT8mJgbl5eViKSoqMkf4RET0EGqNt0e2ZWZ7PDIqKgpffvkljhw5gp49e8pq26FDBwwbNkzyKMnd1Gp1q+6ARUREbZfx+ygoK1Ew+bcVBAFRUVHYu3cvvvrqK7i5ucnuo76+Ht99952sDSGIiIjI9Ew+ohAZGYnk5GTs27cPnTt3RklJCQDA1tZW3HZyzpw56NGjB+Lj4wEAq1atwmOPPYa+ffuirKwM7777LgoKCjB//nxTh0dERAqnFVTQGrPhkhFt2yKTJwqbN28G8Pvzn3dLTEwU32pVWFgIC4v/DWb8+uuviIiIQElJCR555BH4+Pjg2LFjGDRokKnDIyIihdMaOfVgzB4MbZHJE4WmNpRokJmZKfm8du1arF271tShEBERkZH4rgciIlIU418zzREFIiKih1Y9VKg3Yi8EY9q2RcpKi4iIiEgWjigQEZGicOpBHiYKRESkKPUwbvqg3nShtAnKSouIiIhIFo4oEBGRonDqQR4mCkREpCjGvtiJL4UiIiJ6iAlGvipa4OORRERERL/jiAIRESkKpx7kYaJARESKwrdHyqOstIiIiIhk4YgCEREpSr2Rr5k2pm1bxESBiIgUhVMP8igrLSIiIiJZOKJARESKooUFtEb8O9mYtm0REwUiIlKUekGFeiOmD4xp2xYpKy0iIiIiWTiiQEREisLFjPIwUSAiIkURjHx7pMCdGYmIiB5e9VCh3ogXOxnTti1SVlpEREREsnBEgYiIFEUrGLfOQCuYMJg2gIkCEREpitbINQrGtG2LlPVtiYiISBYmCkREpChaqIwuch05cgSTJ0+Gs7MzVCoVUlJSmm2TmZmJ4cOHQ61Wo2/fvkhKSpKcj4uLg0qlkhQPDw9JnerqakRGRqJbt27o1KkTQkJCUFpaKit2JgpERKQoDTszGlPkqqqqgpeXFzZt2mRQ/fz8fEyaNAljx45FTk4OFi9ejPnz5+PQoUOSeoMHD0ZxcbFYjh49Kjm/ZMkS7N+/H7t27UJWVhauXbuGadOmyYqdaxSIiIjMLCgoCEFBQQbXT0hIgJubG1avXg0AGDhwII4ePYq1a9ciMDBQrNe+fXs4Ojrq7KO8vBxbtmxBcnIynnrqKQBAYmIiBg4ciOPHj+Oxxx4zKBaOKBARkaI0LGY0pgBARUWFpNTU1JgsxuzsbAQEBEiOBQYGIjs7W3Ls4sWLcHZ2Rp8+fTB79mwUFhaK506fPo26ujpJPx4eHnB1dW3UT1OYKBARkaJooRK3cb6v8t81Ci4uLrC1tRVLfHy8yWIsKSmBg4OD5JiDgwMqKipw584dAICvry+SkpKQmpqKzZs3Iz8/H08++SRu3bol9mFpaYkuXbo06qekpMTgWDj1QEREdB+KiopgY2Mjflar1S16/bunMoYOHQpfX1/06tULO3fuxLx580x2HSYKRESkKMJ9Prlwd3sAsLGxkSQKpuTo6Njo6YTS0lLY2NjAyspKZ5suXbqgf//+uHTpkthHbW0tysrKJKMKpaWletc16MKpByIiUhSjph2MfPOkofz8/JCRkSE5lpaWBj8/P71tKisrkZeXBycnJwCAj48POnToIOknNzcXhYWFTfZzL44oEBGRorTGzoyVlZXiv/SB3x9/zMnJQdeuXeHq6oqYmBhcvXoVn376KQBg4cKF2LhxI5YtW4bw8HB89dVX2LlzJw4cOCD2sXTpUkyePBm9evXCtWvXEBsbi3bt2mHWrFkAAFtbW8ybNw/R0dHo2rUrbGxs8MILL8DPz8/gJx4AM4woGLIBxL127doFDw8PaDQaDBkyBAcPHjR1WERERK3m1KlTGDZsGIYNGwYAiI6OxrBhw7By5UoAQHFxseSJBTc3Nxw4cABpaWnw8vLC6tWr8fHHH0sejbxy5QpmzZqFAQMGYMaMGejWrRuOHz8Oe3t7sc7atWvxhz/8ASEhIRg9ejQcHR2xZ88eWbGbZURh8ODBSE9P/99F2uu/zLFjxzBr1izEx8fjD3/4A5KTkzFlyhScOXMGnp6e5giPiIgUzNjpg/tp6+/vD0HQ/zape3ddbGhz9uxZvW127NjR7HU1Gg02bdpk8EZPuphljULDBhANxc7OTm/d9evXY8KECXjxxRcxcOBAvP766xg+fDg2btxojtCIiEjhWmML57bMLIlCUxtA3MvQTSXuVlNT02ijCyIiIjI9kycKzW0AcS99m0o0tRlEfHy8ZJMLFxcXk34HIiJ6eLWFpx4eJCZPFIKCgjB9+nQMHToUgYGBOHjwIMrKyrBz506TXSMmJgbl5eViKSoqMlnfRET0cGOiII/ZH4+8dwOIe+nbVKKpzSDUanWL74BFRESkRGbfcOneDSDudT+bShAREd0vjijIY/JEYenSpcjKysLly5dx7NgxTJ06VbIBxJw5cxATEyPW/+tf/4rU1FSsXr0aP/74I+Li4nDq1ClERUWZOjQiIiImCjKZfOqhYQOImzdvwt7eHk888YRkA4jCwkJYWPwvPxk1ahSSk5OxYsUKvPzyy+jXrx9SUlK4hwIREdEDwOSJQnMbQGRmZjY6Nn36dEyfPt3UoRARETUiAEa+FEpZ+K4HIiJSlNbYmbEtY6JARESKwkRBHr5mmoiIiPTiiAIRESkKRxTkYaJARESKwkRBHk49EBERkV4cUSAiIkURBBUEI0YFjGnbFjFRICIiRdFCZdQ+Csa0bYs49UBERER6cUSBiIgUhYsZ5WGiQEREisI1CvJw6oGIiIj04ogCEREpCqce5GGiQEREisKpB3mYKBARkaIIRo4oKC1R4BoFIiIi0osjCkREpCgCAEEwrr2SMFEgIiJF0UIFFXdmNBinHoiIiEgvjigQEZGi8KkHeZgoEBGRomgFFVTcR8FgnHogIiIivTiiQEREiiIIRj71oLDHHpgoEBGRonCNgjyceiAiIiK9OKJARESKwhEFeTiiQEREitLw9khjilxHjhzB5MmT4ezsDJVKhZSUlGbbZGZmYvjw4VCr1ejbty+SkpIk5+Pj4/Hoo4+ic+fO6N69O6ZMmYLc3FxJHX9/f6hUKklZuHChrNiZKBARkaI0LGY0pshVVVUFLy8vbNq0yaD6+fn5mDRpEsaOHYucnBwsXrwY8+fPx6FDh8Q6WVlZiIyMxPHjx5GWloa6ujqMHz8eVVVVkr4iIiJQXFwslnfeeUdW7Jx6ICIiMrOgoCAEBQUZXD8hIQFubm5YvXo1AGDgwIE4evQo1q5di8DAQABAamqqpE1SUhK6d++O06dPY/To0eJxa2trODo63nfsHFEgIiJF+X1UQGVE+b2fiooKSampqTFZjNnZ2QgICJAcCwwMRHZ2tt425eXlAICuXbtKjm/fvh12dnbw9PRETEwMbt++LSsWjigQEZGimGoxo4uLi+R4bGws4uLijAlNVFJSAgcHB8kxBwcHVFRU4M6dO7CyspKc02q1WLx4MR5//HF4enqKx59//nn06tULzs7OOHfuHJYvX47c3Fzs2bPH4FiYKBAREd2HoqIi2NjYiJ/VanWrxRIZGYnz58/j6NGjkuMLFiwQ/zxkyBA4OTlh3LhxyMvLg7u7u0F9M1EgIiJFEf5bjGkPADY2NpJEwZQcHR1RWloqOVZaWgobG5tGowlRUVH48ssvceTIEfTs2bPJfn19fQEAly5dYqJARESkS1vYR8HPzw8HDx6UHEtLS4Ofn99dcQh44YUXsHfvXmRmZsLNza3ZfnNycgAATk5OBsfCRIGIiMjMKisrcenSJfFzfn4+cnJy0LVrV7i6uiImJgZXr17Fp59+CgBYuHAhNm7ciGXLliE8PBxfffUVdu7ciQMHDoh9REZGIjk5Gfv27UPnzp1RUlICALC1tYWVlRXy8vKQnJyMiRMnolu3bjh37hyWLFmC0aNHY+jQoQbHzkSBiIiUxVRzDzKcOnUKY8eOFT9HR0cDAEJDQ5GUlITi4mIUFhaK593c3HDgwAEsWbIE69evR8+ePfHxxx+Lj0YCwObNmwH8vqnS3RITEzF37lxYWloiPT0d69atQ1VVFVxcXBASEoIVK1bIit3kiULv3r1RUFDQ6Phf/vIXnRtNJCUlISwsTHJMrVajurra1KEREREBRk494D7a+vv7Q2hip6Z7d11saHP27Fn9YTSz85OLiwuysrIMjlEfkycK33zzDerr68XP58+fx9NPP43p06frbWNjYyPZdlKlUtY+2kRE1HL4mml5TJ4o2NvbSz6/9dZbcHd3x5gxY/S2UalURu0aRUREROZh1p0Za2trsW3bNoSHhzc5SlBZWYlevXrBxcUFwcHB+P7775vst6amptGOWERERIYwbldGI6ct2iCzJgopKSkoKyvD3Llz9dYZMGAAtm7din379mHbtm3QarUYNWoUrly5ordNfHw8bG1txXLv7lhERER6CSrji4KYNVHYsmULgoKC4OzsrLeOn58f5syZA29vb4wZMwZ79uyBvb09PvzwQ71tYmJiUF5eLpaioiJzhE9ERKR4Zns8sqCgAOnp6bL2kwaADh06YNiwYZLnTe+lVqtbdatMIiJqu7iYUR6zjSgkJiaie/fumDRpkqx29fX1+O6772TtGkVERGQwwQRFQcySKGi1WiQmJiI0NBTt20sHLebMmYOYmBjx86pVq/Cvf/0LP//8M86cOYM//vGPKCgowPz5880RGhEREclglqmH9PR0FBYWIjw8vNG5wsJCWFj8Lz/59ddfERERgZKSEjzyyCPw8fHBsWPHMGjQIHOERkRECtcW3vXwIDFLojB+/Hi9O0ZlZmZKPq9duxZr1641RxhERES6KWz6wBhmfeqBiIiI2ja+FIqIiBSFUw/yMFEgIiJlaYW3R7ZlTBSIiEhhVP8txrRXDq5RICIiIr04okBERMrCqQdZmCgQEZGyMFGQhVMPREREpBdHFIiISFmMfVU0H48kIiJ6ePHtkfJw6oGIiIj04ogCEREpCxczysJEgYiIlIVrFGTh1AMRERHpxREFIiJSFJXwezGmvZIwUSAiImXhGgVZmCgQEZGycI2CLFyjQERERHpxRIGIiJSFUw+yMFEgIiJlYaIgC6ceiIiISC+OKBARkbJwREEWJgpERKQsfOpBFk49EBERkV4cUSAiIkXhzozycESBiIiURTBBkenIkSOYPHkynJ2doVKpkJKS0mybzMxMDB8+HGq1Gn379kVSUlKjOps2bULv3r2h0Wjg6+uLkydPSs5XV1cjMjIS3bp1Q6dOnRASEoLS0lJZsTNRICIiMrOqqip4eXlh06ZNBtXPz8/HpEmTMHbsWOTk5GDx4sWYP38+Dh06JNb54osvEB0djdjYWJw5cwZeXl4IDAzE9evXxTpLlizB/v37sWvXLmRlZeHatWuYNm2arNg59UBERGRmQUFBCAoKMrh+QkIC3NzcsHr1agDAwIEDcfToUaxduxaBgYEAgDVr1iAiIgJhYWFimwMHDmDr1q146aWXUF5eji1btiA5ORlPPfUUACAxMREDBw7E8ePH8dhjjxkUC0cUiIhIUVT43zqF+yr/7aeiokJSampqTBZjdnY2AgICJMcCAwORnZ0NAKitrcXp06cldSwsLBAQECDWOX36NOrq6iR1PDw84OrqKtYxBBMFIiJSlobHI40pAFxcXGBrayuW+Ph4k4VYUlICBwcHyTEHBwdUVFTgzp07uHHjBurr63XWKSkpEfuwtLREly5d9NYxBKceiIiI7kNRURFsbGzEz2q1uhWjMR8mCkREpCwm2pnRxsZGkiiYkqOjY6OnE0pLS2FjYwMrKyu0a9cO7dq101nH0dFR7KO2thZlZWWSUYW76xiCUw9ERKQsrfB4pFx+fn7IyMiQHEtLS4Ofnx8AwNLSEj4+PpI6Wq0WGRkZYh0fHx906NBBUic3NxeFhYViHUNwRIGIiMjMKisrcenSJfFzfn4+cnJy0LVrV7i6uiImJgZXr17Fp59+CgBYuHAhNm7ciGXLliE8PBxfffUVdu7ciQMHDoh9REdHIzQ0FCNGjMDIkSOxbt06VFVViU9B2NraYt68eYiOjkbXrl1hY2ODF154AX5+fgY/8QAwUSAiIoVpjZ0ZT506hbFjx4qfo6OjAQChoaFISkpCcXExCgsLxfNubm44cOAAlixZgvXr16Nnz574+OOPxUcjAWDmzJn45ZdfsHLlSpSUlMDb2xupqamSBY5r166FhYUFQkJCUFNTg8DAQHzwwQcyv68gyPrKR44cwbvvvovTp0+juLgYe/fuxZQpU8TzgiAgNjYWf//731FWVobHH38cmzdvRr9+/Zrsd9OmTXj33XdRUlICLy8vbNiwASNHjjQopoqKCtja2sL1rTdgodHI+TpERPQA0FZXo/ClFSgvLzfbvH/D74reb7xp1O8KbXU1Lq94xayxPkhkr1Fobnepd955B++//z4SEhJw4sQJdOzYEYGBgaiurtbbpyG7SxEREVHLk50oBAUF4Y033sDUqVMbnRMEAevWrcOKFSsQHByMoUOH4tNPP8W1a9ea3Nf67t2lBg0ahISEBFhbW2Pr1q1ywyMiImpaG1jM+CAx6VMP+fn5KCkpkewCZWtrC19fX727QBmyu9S9ampqGu2IRUREZAijdmU0cn1DW2TSRKFhp6emdoq6lyG7S90rPj5eshuWi4uLCaInIiKie7XJfRRiYmJQXl4ulqKiotYOiYiI2goTbeGsFCZ9PLJhp6fS0lI4OTmJx0tLS+Ht7a2zjZ2dXbO7S91LrVY/tFtlEhGRmZloZ0alMOmIgpubGxwdHSW7QFVUVODEiRN6d4EyZHcpIiIiU+EaBXlkjyg0t7vU4sWL8cYbb6Bfv35wc3PDq6++CmdnZ8leC+PGjcPUqVMRFRUFoPndpYiIiKh1yE4UmttdatmyZaiqqsKCBQtQVlaGJ554AqmpqdDctblFXl4ebty4IX42ZHcpIiIik+DUgyyyd2Z8EHFnRiKitq0ld2bs8+r/h3ZG/K6or67Gz6+/zJ0ZiYiIiPhSKCIiUhZOPcjCRIGIiJSFiYIsnHogIiIivTiiQEREimLsXghK20eBIwpERESkFxMFIiIi0otTD0REpCxczCgLEwUiIlIUrlGQh4kCEREpj8J+2RuDaxSIiIhIL44oEBGRsnCNgixMFIiISFG4RkEeTj0QERGRXhxRICIiZeHUgyxMFIiISFE49SAPpx6IiIhIL44oEBGRsnDqQRYmCkREpCxMFGTh1AMRERHpxREFIiJSFC5mlIeJAhERKQunHmRhokBERMrCREEWrlEgIiIivTiiQEREisI1CvIwUSAiImXh1IMsnHogIiJqAZs2bULv3r2h0Wjg6+uLkydP6q1bV1eHVatWwd3dHRqNBl5eXkhNTZXU6d27N1QqVaMSGRkp1vH39290fuHChbLi5ogCEREpSmtMPXzxxReIjo5GQkICfH19sW7dOgQGBiI3Nxfdu3dvVH/FihXYtm0b/v73v8PDwwOHDh3C1KlTcezYMQwbNgwA8M0336C+vl5sc/78eTz99NOYPn26pK+IiAisWrVK/GxtbS0rdo4oEBGRsggmKDKtWbMGERERCAsLw6BBg5CQkABra2ts3bpVZ/3PPvsML7/8MiZOnIg+ffpg0aJFmDhxIlavXi3Wsbe3h6Ojo1i+/PJLuLu7Y8yYMZK+rK2tJfVsbGxkxc5EgYiI6D5UVFRISk1Njc56tbW1OH36NAICAsRjFhYWCAgIQHZ2ts42NTU10Gg0kmNWVlY4evSo3mts27YN4eHhUKlUknPbt2+HnZ0dPD09ERMTg9u3b8v5mkwUiIhIYUw0ouDi4gJbW1uxxMfH67zcjRs3UF9fDwcHB8lxBwcHlJSU6GwTGBiINWvW4OLFi9BqtUhLS8OePXtQXFyss35KSgrKysowd+5cyfHnn38e27Ztw+HDhxETE4PPPvsMf/zjH5u+P/fgGgUiIlIU1X+LMe0BoKioSDKMr1arjQlLYv369YiIiICHhwdUKhXc3d0RFhamd6piy5YtCAoKgrOzs+T4ggULxD8PGTIETk5OGDduHPLy8uDu7m5QLBxRICIiug82NjaSoi9RsLOzQ7t27VBaWio5XlpaCkdHR51t7O3tkZKSgqqqKhQUFODHH39Ep06d0KdPn0Z1CwoKkJ6ejvnz5zcbs6+vLwDg0qVLzdZtwESBiIiUpYUXM1paWsLHxwcZGRniMa1Wi4yMDPj5+TXZVqPRoEePHvjtt9+we/duBAcHN6qTmJiI7t27Y9KkSc3GkpOTAwBwcnIyOH5OPRARkaK0xuOR0dHRCA0NxYgRIzBy5EisW7cOVVVVCAsLAwDMmTMHPXr0ENc5nDhxAlevXoW3tzeuXr2KuLg4aLVaLFu2TNKvVqtFYmIiQkND0b699Fd6Xl4ekpOTMXHiRHTr1g3nzp3DkiVLMHr0aAwdOtTg2GWPKBw5cgSTJ0+Gs7MzVCoVUlJSxHN1dXVYvnw5hgwZgo4dO8LZ2Rlz5szBtWvXmuwzLi6u0YYQHh4eckMjIiJqXis8Hjlz5ky89957WLlyJby9vZGTk4PU1FRxgWNhYaFkoWJ1dTVWrFiBQYMGYerUqejRoweOHj2KLl26SPpNT09HYWEhwsPDG13T0tIS6enpGD9+PDw8PPC3v/0NISEh2L9/v6zYZY8oVFVVwcvLC+Hh4Zg2bZrk3O3bt3HmzBm8+uqr8PLywq+//oq//vWveOaZZ3Dq1Kkm+x08eDDS09P/F1h7DnYQEdHDIyoqClFRUTrPZWZmSj6PGTMGFy5caLbP8ePHQxB0Zy4uLi7IysqSHee9ZP82DgoKQlBQkM5ztra2SEtLkxzbuHEjRo4cicLCQri6uuoPpH17vYs6iIiITEph72swhtkXM5aXl0OlUjUaLrnXxYsX4ezsjD59+mD27NkoLCzUW7empqbRRhdERESGaFijYExRErMmCtXV1Vi+fDlmzZrV5JaRvr6+SEpKQmpqKjZv3oz8/Hw8+eSTuHXrls768fHxkk0uXFxczPUViIiIFM1siUJdXR1mzJgBQRCwefPmJusGBQVh+vTpGDp0KAIDA3Hw4EGUlZVh586dOuvHxMSgvLxcLEVFReb4CkRE9DBqhcWMbZlZVgw2JAkFBQX46quvZL+AokuXLujfv7/eDSHUarVJd8AiIiLlaI3HI9syk48oNCQJFy9eRHp6Orp16ya7j8rKSuTl5cnaEIKIiIhMT3aiUFlZiZycHHF3p/z8fOTk5KCwsBB1dXV49tlncerUKWzfvh319fUoKSlBSUkJamtrxT7GjRuHjRs3ip+XLl2KrKwsXL58GceOHcPUqVPRrl07zJo1y/hvSEREdDdOPcgie+rh1KlTGDt2rPg5OjoaABAaGoq4uDj885//BAB4e3tL2h0+fBj+/v4Aft8t6saNG+K5K1euYNasWbh58ybs7e3xxBNP4Pjx47C3t5cbHhERUZM49SCP7ETB399f7+YOAJo81+Dy5cuSzzt27JAbBhEREbUAbn9IRETKYuz0AUcUiIiIHmJMFGRhokBERIrCNQrymH0LZyIiImq7OKJARETKwqkHWZgoEBGRoqgEASoDntBrqr2ScOqBiIiI9OKIAhERKQunHmRhokBERIrCpx7k4dQDERER6cURBSIiUhZOPcjCRIGIiBSFUw/ycOqBiIiI9OKIAhERKQunHmRhokBERIrCqQd5mCgQEZGycERBFq5RICIiIr04okBERIqjtOkDYzBRICIiZRGE34sx7RWEUw9ERESkF0cUiIhIUfjUgzxMFIiISFn41IMsnHogIiIivTiiQEREiqLS/l6Maa8kTBSIiEhZOPUgC6ceiIiISC8mCkREpCgNTz0YU+7Hpk2b0Lt3b2g0Gvj6+uLkyZN669bV1WHVqlVwd3eHRqOBl5cXUlNTJXXi4uKgUqkkxcPDQ1KnuroakZGR6NatGzp16oSQkBCUlpbKipuJAhERKUvDhkvGFJm++OILREdHIzY2FmfOnIGXlxcCAwNx/fp1nfVXrFiBDz/8EBs2bMCFCxewcOFCTJ06FWfPnpXUGzx4MIqLi8Vy9OhRyfklS5Zg//792LVrF7KysnDt2jVMmzZNVuxMFIiISFFaY0RhzZo1iIiIQFhYGAYNGoSEhARYW1tj69atOut/9tlnePnllzFx4kT06dMHixYtwsSJE7F69WpJvfbt28PR0VEsdnZ24rny8nJs2bIFa9aswVNPPQUfHx8kJibi2LFjOH78uMGxM1EgIiK6DxUVFZJSU1Ojs15tbS1Onz6NgIAA8ZiFhQUCAgKQnZ2ts01NTQ00Go3kmJWVVaMRg4sXL8LZ2Rl9+vTB7NmzUVhYKJ47ffo06urqJNf18PCAq6ur3uvqwkSBiIiURTBBAeDi4gJbW1uxxMfH67zcjRs3UF9fDwcHB8lxBwcHlJSU6GwTGBiINWvW4OLFi9BqtUhLS8OePXtQXFws1vH19UVSUhJSU1OxefNm5Ofn48knn8StW7cAACUlJbC0tESXLl0Mvq4ufDySiIgUxVRbOBcVFcHGxkY8rlarjYzsf9avX4+IiAh4eHhApVLB3d0dYWFhkqmKoKAg8c9Dhw6Fr68vevXqhZ07d2LevHkmi4UjCkRERPfBxsZGUvQlCnZ2dmjXrl2jpw1KS0vh6Oios429vT1SUlJQVVWFgoIC/Pjjj+jUqRP69OmjN54uXbqgf//+uHTpEgDA0dERtbW1KCsrM/i6ujBRICIiZWnhpx4sLS3h4+ODjIwM8ZhWq0VGRgb8/PyabKvRaNCjRw/89ttv2L17N4KDg/XWraysRF5eHpycnAAAPj4+6NChg+S6ubm5KCwsbPa6d+PUAxERKUprvD0yOjoaoaGhGDFiBEaOHIl169ahqqoKYWFhAIA5c+agR48e4jqHEydO4OrVq/D29sbVq1cRFxcHrVaLZcuWiX0uXboUkydPRq9evXDt2jXExsaiXbt2mDVrFgDA1tYW8+bNQ3R0NLp27QobGxu88MIL8PPzw2OPPWZw7LJHFI4cOYLJkyfD2dkZKpUKKSkpkvNz585ttAHEhAkTmu1XzkYUREREbcnMmTPx3nvvYeXKlfD29kZOTg5SU1PFBY6FhYWShYrV1dVYsWIFBg0ahKlTp6JHjx44evSoZGHilStXMGvWLAwYMAAzZsxAt27dcPz4cdjb24t11q5diz/84Q8ICQnB6NGj4ejoiD179siKXfaIQlVVFby8vBAeHq5304YJEyYgMTFR/NzcAo+GjSgSEhLg6+uLdevWITAwELm5uejevbvcEImIiPRrpXc9REVFISoqSue5zMxMyecxY8bgwoULTfa3Y8eOZq+p0WiwadMmbNq0yeA47yU7UQgKCpKstNRFrVbLWihx90YUAJCQkIADBw5g69ateOmll+SGSEREpFdrTD20ZWZZzJiZmYnu3btjwIABWLRoEW7evKm37v1uRHHvRhdERERkeiZPFCZMmIBPP/0UGRkZePvtt5GVlYWgoCDU19frrH8/G1HEx8dLNrlwcXEx9dcgIqKHlVYwviiIyZ96eO6558Q/DxkyBEOHDoW7uzsyMzMxbtw4k1wjJiYG0dHR4ueKigomC0REZJhWWqPQVpl9H4U+ffrAzs5O3ADiXvezEYVarW600QUREZEhVDDypVCt/QVamNkThStXruDmzZviBhD3MmYjCiIiIjIv2YlCZWUlcnJykJOTAwDIz89HTk4OCgsLUVlZiRdffBHHjx/H5cuXkZGRgeDgYPTt2xeBgYFiH+PGjcPGjRvFz9HR0fj73/+OTz75BD/88AMWLVok2YiCiIjIZFp4Z8a2TvYahVOnTmHs2LHi54a1AqGhodi8eTPOnTuHTz75BGVlZXB2dsb48ePx+uuvS/ZSyMvLw40bN8TPM2fOxC+//IKVK1eipKQE3t7eko0oiIiITIWPR8ojO1Hw9/eH0EQ2dejQoWb7uHz5cqNjTW1EQURERK2D73ogIiJl4VMPsjBRICIiRVEJAlRGrDMwpm1bxNdMExERkV4cUSAiImXR/rcY015BmCgQEZGicOpBHk49EBERkV4cUSAiImXhUw+yMFEgIiJlMXZ3RYVNPTBRICIiReHOjPJwjQIRERHpxREFIiJSFk49yMJEgYiIFEWl/b0Y015JOPVAREREenFEgYiIlIVTD7IwUSAiImXhPgqycOqBiIiI9OKIAhERKQrf9SAPEwUiIlIWrlGQhVMPREREpBdHFIiISFkEAMbshaCsAQUmCkREpCxcoyAPEwUiIlIWAUauUTBZJG0C1ygQERGRXhxRICIiZeFTD7IwUSAiImXRAlAZ2V5BOPVAREREejFRICIiRWl46sGYcj82bdqE3r17Q6PRwNfXFydPntRbt66uDqtWrYK7uzs0Gg28vLyQmpoqqRMfH49HH30UnTt3Rvfu3TFlyhTk5uZK6vj7+0OlUknKwoULZcXNRIGIiJSlYY2CMUWmL774AtHR0YiNjcWZM2fg5eWFwMBAXL9+XWf9FStW4MMPP8SGDRtw4cIFLFy4EFOnTsXZs2fFOllZWYiMjMTx48eRlpaGuro6jB8/HlVVVZK+IiIiUFxcLJZ33nlHVuxMFIiIiMxszZo1iIiIQFhYGAYNGoSEhARYW1tj69atOut/9tlnePnllzFx4kT06dMHixYtwsSJE7F69WqxTmpqKubOnYvBgwfDy8sLSUlJKCwsxOnTpyV9WVtbw9HRUSw2NjayYmeiQEREymKiEYWKigpJqamp0Xm52tpanD59GgEBAeIxCwsLBAQEIDs7W2ebmpoaaDQayTErKyscPXpU79cqLy8HAHTt2lVyfPv27bCzs4OnpydiYmJw+/bt5u/RXZgoEBGRspgoUXBxcYGtra1Y4uPjdV7uxo0bqK+vh4ODg+S4g4MDSkpKdLYJDAzEmjVrcPHiRWi1WqSlpWHPnj0oLi7WWV+r1WLx4sV4/PHH4enpKR5//vnnsW3bNhw+fBgxMTH47LPP8Mc//lHW7eLjkURERPehqKhIMoyvVqtN1vf69esREREBDw8PqFQquLu7IywsTO9URWRkJM6fP99oxGHBggXin4cMGQInJyeMGzcOeXl5cHd3NygWjigQEZGyaE1QANjY2EiKvkTBzs4O7dq1Q2lpqeR4aWkpHB0ddbaxt7dHSkoKqqqqUFBQgB9//BGdOnVCnz59GtWNiorCl19+icOHD6Nnz55NfnVfX18AwKVLl5qsdzcmCkREpCgt/XikpaUlfHx8kJGRIR7TarXIyMiAn59fk201Gg169OiB3377Dbt370ZwcLB4ThAEREVFYe/evfjqq6/g5ubWbCw5OTkAACcnJ4Pj59QDEREpSyts4RwdHY3Q0FCMGDECI0eOxLp161BVVYWwsDAAwJw5c9CjRw9xncOJEydw9epVeHt74+rVq4iLi4NWq8WyZcvEPiMjI5GcnIx9+/ahc+fO4noHW1tbWFlZIS8vD8nJyZg4cSK6deuGc+fOYcmSJRg9ejSGDh1qcOyyRxSOHDmCyZMnw9nZGSqVCikpKZLz927s0FDeffddvX3GxcU1qu/h4SE3NCIiogfSzJkz8d5772HlypXw9vZGTk4OUlNTxQWOhYWFkoWK1dXVWLFiBQYNGoSpU6eiR48eOHr0KLp06SLW2bx5M8rLy+Hv7w8nJyexfPHFFwB+H8lIT0/H+PHj4eHhgb/97W8ICQnB/v37ZcUue0ShqqoKXl5eCA8Px7Rp0xqdv3dF5v/7f/8P8+bNQ0hISJP9Dh48GOnp6f8LrD0HO4iIyAy0AqAyYkRBe39to6KiEBUVpfNcZmam5POYMWNw4cKFJvsTmhnZcHFxQVZWlqwYdZH92zgoKAhBQUF6z9+7MGPfvn0YO3aszgUYkkDat9e7qIOIiMhk+PZIWcy6mLG0tBQHDhzAvHnzmq178eJFODs7o0+fPpg9ezYKCwv11q2pqWm00QURERGZnlkThU8++QSdO3fWOUVxN19fXyQlJSE1NRWbN29Gfn4+nnzySdy6dUtn/fj4eMkmFy4uLuYIn4iIHkrGbrbEEQWT2bp1K2bPnt1oG8p7BQUFYfr06Rg6dCgCAwNx8OBBlJWVYefOnTrrx8TEoLy8XCxFRUXmCJ+IiB5GrfBSqLbMbCsGv/76a+Tm5oqrL+Xo0qUL+vfvr3dDCLVabdIdsIiIiEg3s40obNmyBT4+PvDy8pLdtrKyEnl5ebI2hCAiIjKIVjC+KIjsRKGyshI5OTni7k75+fnIycmRLD6sqKjArl27MH/+fJ19jBs3Dhs3bhQ/L126FFlZWbh8+TKOHTuGqVOnol27dpg1a5bc8IiIiJomaI0vCiJ76uHUqVMYO3as+Dk6OhoAEBoaiqSkJADAjh07IAiC3l/0eXl5uHHjhvj5ypUrmDVrFm7evAl7e3s88cQTOH78OOzt7eWGR0RERCYkO1Hw9/dvdpOHBQsWSN5Yda/Lly9LPu/YsUNuGERERPeH+yjIwu0PiYhIWbRGPuKosDUKTBSIiEhZOKIgC18zTURERHpxRIGIiJRFgJEjCiaLpE1gokBERMrCqQdZOPVAREREenFEgYiIlEWrBWDEpklabrhERET08OLUgyyceiAiIiK9OKJARETKwhEFWZgoEBGRsnBnRlk49UBERER6cUSBiIgURRC0EIx4VbQxbdsiJgpERKQsgmDc9AHXKBARET3EBCPXKCgsUeAaBSIiItKLIwpERKQsWi2gMmKdAdcoEBERPcQ49SALpx6IiIhIL44oEBGRoghaLQQjph74eCQREdHDjFMPsnDqgYiIiPTiiAIRESmLVgBUHFEwFBMFIiJSFkEAYMzjkcpKFDj1QERERHpxRIGIiBRF0AoQjJh6EDiiQERE9BATtMaX+7Bp0yb07t0bGo0Gvr6+OHnypN66dXV1WLVqFdzd3aHRaODl5YXU1FTZfVZXVyMyMhLdunVDp06dEBISgtLSUllxM1EgIiJFEbSC0UWuL774AtHR0YiNjcWZM2fg5eWFwMBAXL9+XWf9FStW4MMPP8SGDRtw4cIFLFy4EFOnTsXZs2dl9blkyRLs378fu3btQlZWFq5du4Zp06bJil0lPARjKBUVFbC1tYXrW2/AQqNp7XCIiEgmbXU1Cl9agfLyctjY2JjlGg2/K/xVU9Fe1eG++/lNqEOmsFdWrL6+vnj00UexceNGAIBWq4WLiwteeOEFvPTSS43qOzs745VXXkFkZKR4LCQkBFZWVti2bZtBfZaXl8Pe3h7Jycl49tlnAQA//vgjBg4ciOzsbDz22GMGxf5QrFFoyHW01dWtHAkREd2Phv//bol/u/4m1Bj1YqffUAfg98Tjbmq1Gmq1ulH92tpanD59GjExMeIxCwsLBAQEIDs7W+c1ampqoLnnH75WVlY4evSowX2ePn0adXV1CAgIEOt4eHjA1dVVeYnCrVu3AABX4t5o5UiIiMgYt27dgq2trVn6trS0hKOjI46WHDS6r06dOsHFxUVyLDY2FnFxcY3q3rhxA/X19XBwcJAcd3BwwI8//qiz/8DAQKxZswajR4+Gu7s7MjIysGfPHtTX1xvcZ0lJCSwtLdGlS5dGdUpKSgz+rg9FouDs7IyioiJ07twZKpVKb72Kigq4uLigqKjIbENb5sC4W1ZbjRtou7Ez7pb1IMYtCAJu3boFZ2dns11Do9EgPz8ftbW1RvclCEKj3ze6RhPu1/r16xEREQEPDw+oVCq4u7sjLCwMW7duNdk1DPVQJAoWFhbo2bOnwfVtbGwemL8ccjDultVW4wbabuyMu2U9aHGbayThbhqNptGQvrnZ2dmhXbt2jZ42KC0thaOjo8429vb2SElJQXV1NW7evAlnZ2e89NJL6NOnj8F9Ojo6ora2FmVlZZJRhaauqwufeiAiIjIjS0tL+Pj4ICMjQzym1WqRkZEBPz+/JttqNBr06NEDv/32G3bv3o3g4GCD+/Tx8UGHDh0kdXJzc1FYWNjsde/2UIwoEBERPciio6MRGhqKESNGYOTIkVi3bh2qqqoQFhYGAJgzZw569OiB+Ph4AMCJEydw9epVeHt74+rVq4iLi4NWq8WyZcsM7tPW1hbz5s1DdHQ0unbtChsbG7zwwgvw8/MzeCEjoLBEQa1WIzY21qTzSC2Bcbestho30HZjZ9wtq63G3ZbNnDkTv/zyC1auXImSkhJ4e3sjNTVVXIxYWFgIC4v/DfJXV1djxYoV+Pnnn9GpUydMnDgRn332mWQKobk+AWDt2rWwsLBASEgIampqEBgYiA8++EBW7A/FPgpERERkHlyjQERERHoxUSAiIiK9mCgQERGRXkwUiIiISC8mCkRERKTXQ5coyHnfNwDs2rULHh4e0Gg0GDJkCA4eNH4PcDni4+Px6KOPonPnzujevTumTJmC3NzcJtskJSVBpVJJSkvvNBYXF9coBg8PjybbtPa9BoDevXs3ilulUkne0Ha31rzXR44cweTJk+Hs7AyVSoWUlBTJeUEQsHLlSjg5OcHKygoBAQG4ePFis/3K/Ttiyrjr6uqwfPlyDBkyBB07doSzszPmzJmDa9euNdnn/fy8mTJuAJg7d26jGCZMmNBsv615vwHo/HlXqVR499139fbZEveb2o6HKlGQ+77vY8eOYdasWZg3bx7Onj2LKVOmYMqUKTh//nyLxZyVlYXIyEgcP34caWlpqKurw/jx41FVVdVkOxsbGxQXF4uloKCghSL+n8GDB0tiaHirmS4Pwr0GgG+++UYSc1paGgBg+vTpetu01r2uqqqCl5cXNm3apPP8O++8g/fffx8JCQk4ceIEOnbsiMDAQFQ38RZVuX9HTB337du3cebMGbz66qs4c+YM9uzZg9zcXDzzzDPN9ivn583UcTeYMGGCJIbPP/+8yT5b+34DkMRbXFyMrVu3QqVSISQkpMl+zX2/qQ0RHiIjR44UIiMjxc/19fWCs7OzEB8fr7P+jBkzhEmTJkmO+fr6Cn/+85/NGmdTrl+/LgAQsrKy9NZJTEwUbG1tWy4oHWJjYwUvLy+D6z+I91oQBOGvf/2r4O7uLmi1Wp3nH4R7LQiCAEDYu3ev+Fmr1QqOjo7Cu+++Kx4rKysT1Gq18Pnnn+vtR+7fEVPHrcvJkycFAEJBQYHeOnJ/3oylK+7Q0FAhODhYVj8P4v0ODg4WnnrqqSbrtPT9pgfbQzOi0PBu7rvfu93c+76zs7Ml9YHfX+2pr35LKC8vBwB07dq1yXqVlZXo1asXXFxcEBwcjO+//74lwpO4ePEinJ2d0adPH8yePRuFhYV66z6I97q2thbbtm1DeHh4k28dfRDu9b3y8/NRUlIiuae2trbw9fXVe0/v5+9ISygvL4dKpWr0Ktx7yfl5M5fMzEx0794dAwYMwKJFi3Dz5k29dR/E+11aWooDBw5g3rx5zdZ9EO43PRgemkShqXdz63vvdklJiaz65qbVarF48WI8/vjj8PT01FtvwIAB2Lp1K/bt24dt27ZBq9Vi1KhRuHLlSovF6uvri6SkJKSmpmLz5s3Iz8/Hk08+iVu3bums/6DdawBISUlBWVkZ5s6dq7fOg3CvdWm4b3Lu6f38HTG36upqLF++HLNmzWryLYZyf97MYcKECfj000+RkZGBt99+G1lZWQgKCkJ9fb3O+g/i/f7kk0/QuXNnTJs2rcl6D8L9pgeHot718KCLjIzE+fPnm50L9PPzk7z5a9SoURg4cCA+/PBDvP766+YOEwAQFBQk/nno0KHw9fVFr169sHPnToP+tfIg2LJlC4KCguDs7Ky3zoNwrx9WdXV1mDFjBgRBwObNm5us+yD8vD333HPin4cMGYKhQ4fC3d0dmZmZGDduXIvEYKytW7di9uzZzS7IfRDuNz04HpoRhft537ejo6Os+uYUFRWFL7/8EocPH0bPnj1lte3QoQOGDRuGS5cumSm65nXp0gX9+/fXG8ODdK8BoKCgAOnp6Zg/f76sdg/CvQYg3jc59/R+/o6YS0OSUFBQgLS0tCZHE3Rp7uetJfTp0wd2dnZ6Y3iQ7jcAfP3118jNzZX9Mw88GPebWs9Dkyjcz/u+/fz8JPUBIC0tTdZ7uo0lCAKioqKwd+9efPXVV3Bzc5PdR319Pb777js4OTmZIULDVFZWIi8vT28MD8K9vltiYiK6d++OSZMmyWr3INxrAHBzc4Ojo6PknlZUVODEiRN67+n9/B0xh4Yk4eLFi0hPT0e3bt1k99Hcz1tLuHLlCm7evKk3hgflfjfYsmULfHx84OXlJbvtg3C/qRW19mpKU9qxY4egVquFpKQk4cKFC8KCBQuELl26CCUlJYIgCMKf/vQn4aWXXhLr//vf/xbat28vvPfee8IPP/wgxMbGCh06dBC+++67Fot50aJFgq2trZCZmSkUFxeL5fbt22Kde+N+7bXXhEOHDgl5eXnC6dOnheeee07QaDTC999/32Jx/+1vfxMyMzOF/Px84d///rcQEBAg2NnZCdevX9cZ84NwrxvU19cLrq6uwvLlyxude5Du9a1bt4SzZ88KZ8+eFQAIa9asEc6ePSs+HfDWW28JXbp0Efbt2yecO3dOCA4OFtzc3IQ7d+6IfTz11FPChg0bxM/N/R0xd9y1tbXCM888I/Ts2VPIycmR/MzX1NTojbu5nzdzx33r1i1h6dKlQnZ2tpCfny+kp6cLw4cPF/r16ydUV1frjbu173eD8vJywdraWti8ebPOPlrjflPb8VAlCoIgCBs2bBBcXV0FS0tLYeTIkcLx48fFc2PGjBFCQ0Ml9Xfu3Cn0799fsLS0FAYPHiwcOHCgReMFoLMkJibqjXvx4sXid3RwcBAmTpwonDlzpkXjnjlzpuDk5CRYWloKPXr0EGbOnClcunRJb8yC0Pr3usGhQ4cEAEJubm6jcw/SvT58+LDOn42G+LRarfDqq68KDg4OglqtFsaNG9foO/Xq1UuIjY2VHGvq74i5487Pz9f7M3/48GG9cTf382buuG/fvi2MHz9esLe3Fzp06CD06tVLiIiIaPQL/0G73w0+/PBDwcrKSigrK9PZR2vcb2o7VIIgCGYdsiAiIqI266FZo0BERESmx0SBiIiI9GKiQERERHoxUSAiIiK9mCgQERGRXkwUiIiISC8mCkRERKQXEwUiIiLSi4kCERER6cVEgYiIiPRiokBERER6/f+ozI4uFWM13gAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAGzCAYAAAC2DMSCAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASz5JREFUeJzt3XtcVGX+B/DPDMIMIhcV5WIoAipeoUBZzBQTBZbNS62S6waSaXkpazZT1/KS+ptcSy0laXe95T1fobbmUjqJl8BcL6SW4SUUSEGxBKEEd+b5/dFycGJAhpkBnPN57+t5vZpznufMd46w8+W5HYUQQoCIiIhkR9nUARAREVHTYBJAREQkU0wCiIiIZIpJABERkUwxCSAiIpIpJgFEREQyxSSAiIhIppgEEBERyRSTACIiIpliEkCyNn78ePj7+zd1GI0iKioKUVFRTfb+CoUC06ZNa7L3J6KamARQk8jNzcW0adPQtWtXtGzZEi1btkSPHj0wdepUnD59uqnDq1VpaSkWLFiAkJAQtGrVCs7OzujVqxdmzpyJq1evmmwzZswYKBQKzJw50+T5jIwMKBQKKBQKbNq0yWSdRx99FAqFAr169bLaZ7GFzMxMzJ8/H7du3WqS9z9x4gT+8Ic/wNvbG61atUKfPn3w3nvvQa/XN0k8RM0dkwBqdHv27EGvXr2wceNGREdHY/ny5Xj33XcRFxeHvXv3IjQ0FFeuXGnqMGv4/vvvERoaioULF6JHjx5YsmQJ3nvvPQwePBhr1qwx+Vd2aWkp/vWvf8Hf3x9bt25FXY/qUKvV2LJlS43jly9fRmZmJtRqtTU/jk1kZmZiwYIFTZIEnDhxAv3798fly5cxc+ZMvPPOOwgICMD06dOh0WgaPR6iB0GLpg6A5OXSpUt4+umn0alTJ+h0Ovj4+BidX7JkCd5//30olXXnp+Xl5XBxcbFlqEb++9//4sknn0RRUREyMjIwYMAAo/OLFy/GkiVLarT7+OOPodfrsXbtWjz++OM4dOgQBg0aZPI9fv/73+OTTz5BcXExPD09peNbtmyBl5cXunTpgp9++sm6H8yOfPDBBwCAQ4cOoU2bNgCA559/HoMGDcL69evx7rvvNmV4RM0SewKoUf3tb39DeXk51q1bVyMBAIAWLVrgpZdegp+fn3Rs/PjxaNWqFS5duoTf//73cHV1xbhx4wAAhw8fxujRo9GxY0eoVCr4+fnhlVdewS+//FLj2rt27UKvXr2gVqvRq1cv7Ny5s95xf/zxx/j6668xZ86cGgkAALi5uWHx4sU1jm/evBlDhw7F4MGD0b17d2zevLnW9xgxYgRUKhV27NhhdHzLli0YM2YMHBwc6h3v3//+dwQGBsLZ2Rn9+vXD4cOHTdarqKjAvHnzEBQUJN2/1157DRUVFUb1qsbzN2/ejG7dukGtViMsLAyHDh2S6syfPx8zZswAAHTu3Fka4rh8+bLRtar+HVQqFXr27In09PR6f666lJaWQq1Ww8PDw+i4j48PnJ2drfIeRPaGPQHUqPbs2YOgoCBERESY1e6///0vYmJiMGDAALz99tto2bIlAGDHjh34+eefMXnyZLRt2xbHjh3DypUrUVBQYPRl+vnnn+Opp55Cjx49oNVqcfPmTSQnJ+Ohhx6q1/t/8sknAIBnnnmm3jFfvXoVBw4cwIYNGwAAY8eOxfLly7Fq1So4OTnVqN+yZUuMGDECW7duxeTJkwEAX3/9Nb755hv885//rPdciTVr1uD5559H//798fLLL+P777/H8OHD0aZNG6PkymAwYPjw4Thy5AgmTZqE7t2748yZM1i+fDnOnz+PXbt2GV334MGD2L59O1566SWoVCq8//77iI2NxbFjx9CrVy88+eSTOH/+PLZu3Yrly5dLvRnt2rWTrnHkyBGkpaVhypQpcHV1xXvvvYennnoKeXl5aNu2LQDg7t27KCkpqddnbdOmjdRrFBUVhe3bt+P555+HRqNBy5Yt8e9//xtpaWlYunRpva5HJDuCqJGUlJQIAGLkyJE1zv3000/ixo0bUvn555+lc0lJSQKAmDVrVo1299arotVqhUKhEFeuXJGOhYaGCh8fH3Hr1i3p2Oeffy4AiE6dOt039ocffli4u7vft9693n77beHs7CxKS0uFEEKcP39eABA7d+40qnfgwAEBQOzYsUPs2bNHKBQKkZeXJ4QQYsaMGSIgIEAIIcSgQYNEz54963zPyspK0b59exEaGioqKiqk43//+98FADFo0CDp2MaNG4VSqRSHDx82ukZqaqoAIL788kvpGAABQBw/flw6duXKFaFWq8WoUaOkY0uXLhUARG5ubo3YAAgnJydx8eJF6djXX38tAIiVK1fWuB/1Kfe+z3//+18xbdo04ejoKJ13cHAQq1evrvOeEckZewKo0ZSWlgIAWrVqVeNcVFQUvv76a+n10qVL8eqrrxrVqfrr+F73dvOWl5fjl19+Qf/+/SGEwKlTp9CxY0dcu3YN2dnZmDVrFtzd3aX6Q4cORY8ePVBeXl6v2F1dXe//Ie+xefNmxMfHS+26dOmCsLAwbN68GSNHjjTZZtiwYWjTpg22bduGV199Fdu2bUNiYmK93/P48eO4fv063nzzTaPehvHjx0td9VV27NiB7t27Izg4GMXFxdLxxx9/HABw4MAB9O/fXzoeGRmJsLAw6XXHjh0xYsQI/Otf/4Jer6/XcEV0dDQCAwOl13369IGbmxu+//576VhISAj27dtXr8/r7e0t/beDgwMCAwMRExOD0aNHQ61WY+vWrXjxxRfh7e1d6z0nkjMmAdRoqr4My8rKapz74IMPcPv2bRQVFeHPf/5zjfMtWrQw2XWfl5eHuXPn4pNPPqkxaa6qS7lqpUGXLl1qtO/WrRtOnjwpvb5x44bRcrJWrVqhVatWNb6o7ufcuXM4deoUEhMTcfHiRel4VFQUUlJSUFpaCjc3txrtHB0dMXr0aGzZsgX9+vVDfn4+/vSnP9X7fWv7rI6OjggICDA6duHCBZw7d86ou/5e169fN3pt6v517doVP//8M27cuGH0hVybjh071jjWunVro3+71q1bIzo6+r7X+q233noL7777Li5cuCAlmmPGjMHgwYMxdepU/OEPf0CLFvy/PKJ78TeCGo27uzt8fHxw9uzZGueq5gj8dhJZFZVKVWPFgF6vx9ChQ/Hjjz9i5syZCA4OhouLC3744QeMHz8eBoPB7Bj79u1rtDxx3rx5mD9/PoKDg3Hq1Cnk5+cbjavXpmq9/yuvvIJXXnmlxvmPP/4YycnJJtv+6U9/QmpqKubPn4+QkBD06NHD7M9RHwaDAb1798ayZctMnq/P5zRXbb0F4p6lk5WVlfjxxx/rdb127dpJ13z//ffx+OOP1+hpGj58ODQaDS5fvoygoKAGRk5kn5gEUKOKj4/HP//5Txw7dgz9+vWz6FpnzpzB+fPnsWHDBqMu8992JXfq1AnAr3/5/lZOTo7R682bNxutLKj66/mJJ57A1q1bsWnTJsyePbvOuIQQ2LJlCwYPHowpU6bUOL9w4UJs3ry51iRgwIAB6NixIzIyMkwuO6zLvZ+1qlsf+HWyXW5uLkJCQqRjgYGB+PrrrzFkyBAoFIr7XtvU/Tt//jxatmwp9SbU5zr3k5mZicGDB9erbm5urrTjY1FRkclNge7evQvg18mlRGSMSQA1qtdeew1btmzBs88+C51OBy8vL6Pzoo7NdH6r6i/Ae9sIIWqsB/fx8UFoaCg2bNhgNC9g3759+Pbbb6UvTuDXnflM+eMf/witVovFixcjKioKkZGRRudv376Nt956C4sXL8aXX36Jy5cv480338Qf//jHGtc6f/483njjDVy9ehW+vr41zisUCrz33ns4deqUWasRACA8PBzt2rVDamoqkpOTpXkB69evr7GBz5gxY7B371784x//wKRJk4zO/fLLLzAYDEZ7MWRlZeHkyZN45JFHAAD5+fnYvXs3YmNjpX+LqvqWbBbU0DkBXbt2xb59+3Dz5k1ppYFer8dHH30EV1dXo7kIRPQrJgHUqLp06YItW7Zg7Nix6NatG8aNG4eQkBAIIZCbm4stW7ZAqVTWa+lecHAwAgMD8eqrr+KHH36Am5sbPv74Y5Mb6mi1WsTHx2PAgAF49tln8eOPP2LlypXo2bOnyTkKv+Xo6Ii0tDRER0dj4MCBGDNmDB599FE4Ojrim2++wZYtW9C6dWssXrwYmzdvhoODA+Lj401ea/jw4ZgzZw62bdtW6052I0aMwIgRI+4bl6k4Fy1ahOeffx6PP/44EhISkJubi3Xr1tWYE/DMM8/go48+wgsvvIADBw7g0UcfhV6vx3fffYePPvoIn332GcLDw6X6vXr1QkxMjNESQQBYsGCBVKdq4uCcOXPw9NNPw9HREU888YRZGzs1dE7ArFmz8Oc//xkRERGYNGkSnJ2dsXXrVpw4cQKLFi2Co6Oj2dcksntNuDKBZOzixYti8uTJIigoSKjVauHs7CyCg4PFCy+8ILKzs43qJiUlCRcXF5PX+fbbb0V0dLRo1aqV8PT0FBMnTpSWna1bt86o7scffyy6d+8uVCqV6NGjh0hLSxNJSUn1WiJY5aeffhJz584VvXv3Fi1bthRqtVr06tVLzJ49W1y7dk1UVlaKtm3biscee6zO63Tu3Fk8/PDDQgjjJYJ1qc8SwSrvv/++6Ny5s1CpVCI8PFwcOnRIDBo0yGiJoBC/LilcsmSJ6Nmzp1CpVKJ169YiLCxMLFiwQJSUlEj1AIipU6eKTZs2iS5dugiVSiUefvhhceDAgRrvvXDhQtGhQwehVCqNlvFVXeO3OnXqJJKSkur1ue4nPT1dDBo0SHh6egonJyfRu3dvkZqaapVrE9kjhRBm9L8SkSwpFApMnToVq1ataupQiMiKuG0wERGRTDEJICIikikmAURERDLF1QFEdF+cOkRkn9gTQEREJFNMAoiIiGTKLoYDDAYDrl69CldXV6tsW0pERI1LCIHbt2/D19e3xnNCrOnOnTuorKy0+DpOTk5Qq9VWiKhp2UUScPXqVZs87ISIiBpXfn5+vXYMbYg7d+6gc6dWKLxe8xkT5vL29kZubu4DnwjYRRJQ9Yjah+a/DuUD/g9CRCRHhjt3UDB/kfT/57ZQWVmJwut65J7oBDfXhvc2lN42oHPYFVRWVjIJaA6qhgCUajWTACKiB1hjDOm6uSotSgLsiV0kAURERPWlFwboLVj1qhcG6wXTxJgEEBGRrBggYEDDswBL2jY3NusPSUlJgb+/P9RqNSIiInDs2LE66+/YsQPBwcFQq9Xo3bs39u7da6vQiIhIxgxW+J+9sEkSsH37dmg0GsybNw8nT55ESEgIYmJicP36dZP1MzMzMXbsWEyYMAGnTp3CyJEjMXLkSJw9e9YW4RERERFslAQsW7YMEydORHJyMnr06IHU1FS0bNkSa9euNVn/3XffRWxsLGbMmIHu3btj4cKFeOSRR/jYUiIisjq9EBYXe2H1JKCyshInTpxAdHR09ZsolYiOjkZWVpbJNllZWUb1ASAmJqbW+hUVFSgtLTUqRERE9VE1J8CSYi+sngQUFxdDr9fDy8vL6LiXlxcKCwtNtiksLDSrvlarhbu7u1S4URAREZH5HsiFkrNnz0ZJSYlU8vPzmzokIiJ6QBggoLeg2FNPgNWXCHp6esLBwQFFRUVGx4uKiuDt7W2yjbe3t1n1VSoVVCqVdQImIiJZ4RLBalbvCXByckJYWBh0Op10zGAwQKfTITIy0mSbyMhIo/oAsG/fvlrrExERkeVsMhyg0Wjwj3/8Axs2bMC5c+cwefJklJeXIzk5GQCQmJiI2bNnS/WnT5+O9PR0vPPOO/juu+8wf/58HD9+HNOmTbNFeEREJGNNtTrAnP1z0tLSEB4eDg8PD7i4uCA0NBQbN240qlNUVITx48fD19cXLVu2RGxsLC5cuGBWTDbZMTAhIQE3btzA3LlzUVhYiNDQUKSnp0uT//Ly8oweFdm/f39s2bIFr7/+Ov7617+iS5cu2LVrF3r16mWL8IiISMYM/yuWtDdX1f45qampiIiIwIoVKxATE4OcnBy0b9++Rv02bdpgzpw5CA4OhpOTE/bs2YPk5GS0b98eMTExEEJg5MiRcHR0xO7du+Hm5oZly5YhOjoa3377LVxcXOoVl0KIB3/BY2lpKdzd3dHxrUV8gBAR0QPIcOcO8ma9jpKSEri5udnkPaq+K7475wVXCx4gdPu2AcHdi8yKNSIiAn379pX2vzEYDPDz88OLL76IWbNm1esajzzyCOLj47Fw4UKcP38e3bp1w9mzZ9GzZ0/pmt7e3vi///s/PPfcc/W65gO5OoCIiKihLFkZUFUA1NivpqKiwuT7NWT/nHsJIaDT6ZCTk4OBAwcCgPRe9z7KWKlUQqVS4ciRI/W+F0wCiIhIVvTC8gIAfn5+RnvWaLVak+/XkP1zAKCkpAStWrWCk5MT4uPjsXLlSgwdOhQAEBwcjI4dO2L27Nn46aefUFlZiSVLlqCgoADXrl2r973gUwSJiEhWrDUnID8/32g4wNpL111dXZGdnY2ysjLodDpoNBoEBAQgKioKjo6OSEtLw4QJE9CmTRs4ODggOjoacXFxMGeUn0kAERFRA7i5udVrTkBD9s8Bfu3eDwoKAgCEhobi3Llz0Gq1iIqKAgCEhYUhOzsbJSUlqKysRLt27RAREYHw8PB6fwYOBxARkawYoIDegmKAwqz3a8j+OSbjNhhMzjtwd3dHu3btcOHCBRw/fhwjRoyo9zXZE0BERLJiEL8WS9qbS6PRICkpCeHh4ejXrx9WrFhRY/+cDh06SPMKtFotwsPDERgYiIqKCuzduxcbN27E6tWrpWvu2LED7dq1Q8eOHXHmzBlMnz4dI0eOxLBhw+odF5MAIiIiGzN3/5zy8nJMmTIFBQUFcHZ2RnBwMDZt2oSEhASpzrVr16DRaFBUVAQfHx8kJibijTfeMCsu7hNARERNrjH3CfjqG2+0smCfgLLbBkT0LLRprI2FPQFERCQrVWP7lrS3F5wYSEREJFPsCSAiIlkxCAUMouF/zVvStrlhEkBERLLC4YBqHA4gIiKSKfYEEBGRrOihhN6Cv4H1VoylqTEJICIiWREWzgkQnBNARET0YOKcgGqcE0BERCRT7AkgIiJZ0Qsl9MKCOQEP/D671ZgEEBGRrBiggMGCjnAD7CcL4HAAERGRTLEngIiIZIUTA6sxCSAiIlmxfE4AhwOIiIjoAceeACIikpVfJwZa8AAhDgcQERE9mAwWbhvM1QFERET0wGNPABERyQonBlZjEkBERLJigJKbBf0PkwAiIpIVvVBAb8GTAC1p29xwTgAREZFMsSeAiIhkRW/h6gA9hwOIiIgeTAahhMGCiYEGO5oYyOEAIiIimWJPABERyQqHA6oxCSAiIlkxwLIZ/gbrhdLkrD4coNVq0bdvX7i6uqJ9+/YYOXIkcnJy6myzfv16KBQKo6JWq60dGhEREd3D6knAwYMHMXXqVBw9ehT79u3D3bt3MWzYMJSXl9fZzs3NDdeuXZPKlStXrB0aERGRtFmQJcVeWH04ID093ej1+vXr0b59e5w4cQIDBw6stZ1CoYC3t7e1wyEiIjJi+bbB9pME2PyTlJSUAADatGlTZ72ysjJ06tQJfn5+GDFiBL755pta61ZUVKC0tNSoEBERkXlsmgQYDAa8/PLLePTRR9GrV69a63Xr1g1r167F7t27sWnTJhgMBvTv3x8FBQUm62u1Wri7u0vFz8/PVh+BiIjsjAEKi4u9sGkSMHXqVJw9exbbtm2rs15kZCQSExMRGhqKQYMGIS0tDe3atcMHH3xgsv7s2bNRUlIilfz8fFuET0REdqhqOMCSYi9stkRw2rRp2LNnDw4dOoSHHnrIrLaOjo54+OGHcfHiRZPnVSoVVCqVNcIkIiKZsXyfAPtJAqz+SYQQmDZtGnbu3IkvvvgCnTt3Nvsaer0eZ86cgY+Pj7XDIyIiov+xek/A1KlTsWXLFuzevRuurq4oLCwEALi7u8PZ2RkAkJiYiA4dOkCr1QIA3nzzTfzud79DUFAQbt26haVLl+LKlSt47rnnrB0eERHJnEEoYLBksyA7epSw1ZOA1atXAwCioqKMjq9btw7jx48HAOTl5UGprO6E+OmnnzBx4kQUFhaidevWCAsLQ2ZmJnr06GHt8IiISOYMFg4HcJ+AOoh6PF0pIyPD6PXy5cuxfPlya4dCREREdeCzA4iISFYsf5QwewKIiIgeSHoooLdgrb8lbZsb+0lniIiIyCxMAoiISFaqhgMsKQ2RkpICf39/qNVqRERE4NixY7XWTUtLQ3h4ODw8PODi4oLQ0FBs3LjRqE5ZWRmmTZuGhx56CM7OzujRowdSU1PNionDAUREJCt6WNalr29Am+3bt0Oj0SA1NRURERFYsWIFYmJikJOTg/bt29eo36ZNG8yZMwfBwcFwcnLCnj17kJycjPbt2yMmJgYAoNFo8MUXX2DTpk3w9/fH559/jilTpsDX1xfDhw+vV1zsCSAiIrKxZcuWYeLEiUhOTpb+Ym/ZsiXWrl1rsn5UVBRGjRqF7t27IzAwENOnT0efPn1w5MgRqU5mZiaSkpIQFRUFf39/TJo0CSEhIXX2MPwWkwAiIpIVaw0H/PZpthUVFSbfr7KyEidOnEB0dLR0TKlUIjo6GllZWfeNVwgBnU6HnJwcDBw4UDrev39/fPLJJ/jhhx8ghMCBAwdw/vx5DBs2rN73gkkAERHJirUeIOTn52f0RNuqXXB/q7i4GHq9Hl5eXkbHvby8pF11TSkpKUGrVq3g5OSE+Ph4rFy5EkOHDpXOr1y5Ej169MBDDz0EJycnxMbGIiUlxShRuB/OCSAiIlkRFj4OWPyvbX5+Ptzc3KTj1n6wnaurK7Kzs1FWVgadTgeNRoOAgABpR96VK1fi6NGj+OSTT9CpUyccOnQIU6dOha+vr1GvQ12YBBARETWAm5ubURJQG09PTzg4OKCoqMjoeFFREby9vWttp1QqERQUBAAIDQ3FuXPnoNVqERUVhV9++QV//etfsXPnTsTHxwMA+vTpg+zsbLz99tv1TgI4HEBERLJireGA+nJyckJYWBh0Op10zGAwQKfTITIyst7XMRgM0ryDu3fv4u7du0bP4QEABwcHGAyGel+TPQFERCQrTfEUQY1Gg6SkJISHh6Nfv35YsWIFysvLkZycDKDm03W1Wi3Cw8MRGBiIiooK7N27Fxs3bpQe0ufm5oZBgwZhxowZcHZ2RqdOnXDw4EF8+OGHWLZsWb3jYhJARERkYwkJCbhx4wbmzp2LwsJChIaGIj09XZos+Nun65aXl2PKlCkoKCiAs7MzgoODsWnTJiQkJEh1tm3bhtmzZ2PcuHH48ccf0alTJyxevBgvvPBCveNSiPo89q+ZKy0thbu7Ozq+tQhKtbqpwyEiIjMZ7txB3qzXUVJSUq9x9oao+q54+cvhULVybPB1KsruYsWjn9g01sbCngAiIpKVphgOaK44MZCIiEim2BNARESyYoASBgv+BrakbXPDJICIiGRFLxTQW9Clb0nb5sZ+0hkiIiIyC3sCiIhIVjgxsBqTACIikhVxz5MAG9reXjAJICIiWdFDAb0FDxCypG1zYz/pDBEREZmFPQFERCQrBmHZuL7hgd9ntxqTACIikhWDhXMCLGnb3NjPJyEiIiKzsCeAiIhkxQAFDBZM7rOkbXPDJICIiGSFOwZW43AAERGRTLEngIiIZIUTA6sxCSAiIlkxwMJtg+1oToD9pDNERERkFvYEEBGRrAgLVwcIO+oJYBJARESywqcIVmMSQEREssKJgdWs/knmz58PhUJhVIKDg+tss2PHDgQHB0OtVqN3797Yu3evtcMiIiKi37BJOtOzZ09cu3ZNKkeOHKm1bmZmJsaOHYsJEybg1KlTGDlyJEaOHImzZ8/aIjQiIpK5quEAS4q9sEkS0KJFC3h7e0vF09Oz1rrvvvsuYmNjMWPGDHTv3h0LFy7EI488glWrVtkiNCIikrmqbYMtKfbCJknAhQsX4Ovri4CAAIwbNw55eXm11s3KykJ0dLTRsZiYGGRlZdXapqKiAqWlpUaFiIiIzGP1JCAiIgLr169Heno6Vq9ejdzcXDz22GO4ffu2yfqFhYXw8vIyOubl5YXCwsJa30Or1cLd3V0qfn5+Vv0MRERkvzgcUM3qSUBcXBxGjx6NPn36ICYmBnv37sWtW7fw0UcfWe09Zs+ejZKSEqnk5+db7dpERGTfmARUs/kSQQ8PD3Tt2hUXL140ed7b2xtFRUVGx4qKiuDt7V3rNVUqFVQqlVXjJCIikhubL3YsKyvDpUuX4OPjY/J8ZGQkdDqd0bF9+/YhMjLS1qEREZEMsSegmtWTgFdffRUHDx7E5cuXkZmZiVGjRsHBwQFjx44FACQmJmL27NlS/enTpyM9PR3vvPMOvvvuO8yfPx/Hjx/HtGnTrB0aERERk4B7WH04oKCgAGPHjsXNmzfRrl07DBgwAEePHkW7du0AAHl5eVAqq3OP/v37Y8uWLXj99dfx17/+FV26dMGuXbvQq1cva4dGRERE97B6ErBt27Y6z2dkZNQ4Nnr0aIwePdraoRAREdUgYNnjgIX1QmlyfHYAERHJCh8gVI1JABERyQqTgGr28ygkIiIiMgt7AoiISFbYE1CNSQAREckKk4BqHA4gIiKSKfYEEBGRrAihgLDgr3lL2jY3TAKIiEhWDFBYtE+AJW2bGw4HEBERyRSTACIikpWmenZASkoK/P39oVarERERgWPHjtVaNy0tDeHh4fDw8ICLiwtCQ0OxceNGozoKhcJkWbp0ab1jYhJARESyUjUnwJJiru3bt0Oj0WDevHk4efIkQkJCEBMTg+vXr5us36ZNG8yZMwdZWVk4ffo0kpOTkZycjM8++0yqc+3aNaOydu1aKBQKPPXUU/WOi3MCiIiIGqC0tNTotUqlgkqlMll32bJlmDhxIpKTkwEAqamp+PTTT7F27VrMmjWrRv2oqCij19OnT8eGDRtw5MgRxMTEAAC8vb2N6uzevRuDBw9GQEBAvT8DewKIiEhWrDUc4OfnB3d3d6lotVqT71dZWYkTJ04gOjpaOqZUKhEdHY2srKz7xiuEgE6nQ05ODgYOHGiyTlFRET799FNMmDDBrHvBngAiIpIVay0RzM/Ph5ubm3S8tl6A4uJi6PV6eHl5GR338vLCd999V+v7lJSUoEOHDqioqICDgwPef/99DB061GTdDRs2wNXVFU8++aRZn4VJABERyYqwcMfAqiTAzc3NKAmwNldXV2RnZ6OsrAw6nQ4ajQYBAQE1hgoAYO3atRg3bhzUarVZ78EkgIiIyIY8PT3h4OCAoqIio+NFRUU1xvXvpVQqERQUBAAIDQ3FuXPnoNVqayQBhw8fRk5ODrZv3252bJwTQEREsiIACGFBMfP9nJycEBYWBp1OJx0zGAzQ6XSIjIys93UMBgMqKipqHF+zZg3CwsIQEhJiZmTsCSAiIpkxQAFFI+8YqNFokJSUhPDwcPTr1w8rVqxAeXm5tFogMTERHTp0kCYXarVahIeHIzAwEBUVFdi7dy82btyI1atXG123tLQUO3bswDvvvNOgz8IkgIiIyMYSEhJw48YNzJ07F4WFhQgNDUV6ero0WTAvLw9KZXXnfHl5OaZMmYKCggI4OzsjODgYmzZtQkJCgtF1t23bBiEExo4d26C4FEIIc3s2mp3S0lK4u7uj41uLoDRzUgQRETU9w507yJv1OkpKSmw22a7qu6LPjlfh0NL0TP760P9cgdOj37ZprI2FPQFERCQrBqGAwoLVAZasLGhuODGQiIhIptgTQEREslI1y9+S9vaCSQAREcmKtXYMtAccDiAiIpIp9gQQEZGssCegGpMAIiKSFa4OqMYkgIiIZIUTA6txTgAREZFMsSeAiIhk5deeAEvmBFgxmCbGJICIiGSFEwOrcTiAiIhIptgTQEREsiL+Vyxpby+YBBARkaxwOKAahwOIiIhkij0BREQkLxwPkFi9J8Df3x8KhaJGmTp1qsn669evr1FXrVZbOywiIqJf/W84oKEFdjQcYPWegP/85z/Q6/XS67Nnz2Lo0KEYPXp0rW3c3NyQk5MjvVYo7OcGExFR88IdA6tZPQlo166d0eu33noLgYGBGDRoUK1tFAoFvL29rR0KERER1cGmEwMrKyuxadMmPPvss3X+dV9WVoZOnTrBz88PI0aMwDfffFPndSsqKlBaWmpUiIiI6sOSoQBLVxY0NzZNAnbt2oVbt25h/Pjxtdbp1q0b1q5di927d2PTpk0wGAzo378/CgoKam2j1Wrh7u4uFT8/PxtET0REdqlqXN+SYidsmgSsWbMGcXFx8PX1rbVOZGQkEhMTERoaikGDBiEtLQ3t2rXDBx98UGub2bNno6SkRCr5+fm2CJ+IiMiu2WyJ4JUrV7B//36kpaWZ1c7R0REPP/wwLl68WGsdlUoFlUplaYhERCRDnBhYzWY9AevWrUP79u0RHx9vVju9Xo8zZ87Ax8fHRpEREZGsCSsUO2GTJMBgMGDdunVISkpCixbGnQ2JiYmYPXu29PrNN9/E559/ju+//x4nT57En//8Z1y5cgXPPfecLUIjIiKi/7HJcMD+/fuRl5eHZ599tsa5vLw8KJXVucdPP/2EiRMnorCwEK1bt0ZYWBgyMzPRo0cPW4RGREQyx2cHVLNJEjBs2DCIWgZNMjIyjF4vX74cy5cvt0UYREREptlRl74l+AAhIiIimeIDhIiISFY4HFCNSQAREckLnyIoYRJAREQyo/hfsaS9feCcACIiIpliTwAREckLhwMkTAKIiEhemARIOBxAREQkU+wJICIiebH0ccBcIkhERPRg4lMEq3E4gIiISKbYE0BERPLCiYESJgFERCQvnBMg4XAAERFRI0hJSYG/vz/UajUiIiJw7NixWuumpaUhPDwcHh4ecHFxQWhoKDZu3Fij3rlz5zB8+HC4u7vDxcUFffv2RV5eXr1jYhJARESyohCWF3Nt374dGo0G8+bNw8mTJxESEoKYmBhcv37dZP02bdpgzpw5yMrKwunTp5GcnIzk5GR89tlnUp1Lly5hwIABCA4ORkZGBk6fPo033ngDarXajHshHvx5jqWlpXB3d0fHtxZBacaHJyKi5sFw5w7yZr2OkpISuLm52eQ9qr4r/Fa8CaVzw78rDL/cQf7Lc82KNSIiAn379sWqVat+vYbBAD8/P7z44ouYNWtWva7xyCOPID4+HgsXLgQAPP3003B0dDTZQ1Bf7AkgIiJ5qZoTYEnBr0nFvaWiosLk21VWVuLEiROIjo6WjimVSkRHRyMrK+v+4QoBnU6HnJwcDBw4EMCvScSnn36Krl27IiYmBu3bt0dERAR27dpl1q1gEkBERNQAfn5+cHd3l4pWqzVZr7i4GHq9Hl5eXkbHvby8UFhYWOv1S0pK0KpVKzg5OSE+Ph4rV67E0KFDAQDXr19HWVkZ3nrrLcTGxuLzzz/HqFGj8OSTT+LgwYP1/gxcHUBERPJipSWC+fn5RsMBKpXKorB+y9XVFdnZ2SgrK4NOp4NGo0FAQACioqJgMBgAACNGjMArr7wCAAgNDUVmZiZSU1MxaNCger0HkwAiIpIXKyUBbm5u9ZoT4OnpCQcHBxQVFRkdLyoqgre3d63tlEolgoKCAPz6BX/u3DlotVpERUXB09MTLVq0QI8ePYzadO/eHUeOHKn3R+FwABERkQ05OTkhLCwMOp1OOmYwGKDT6RAZGVnv6xgMBmnegZOTE/r27YucnByjOufPn0enTp3qfU32BBARkbw0wY6BGo0GSUlJCA8PR79+/bBixQqUl5cjOTkZAJCYmIgOHTpI8wq0Wi3Cw8MRGBiIiooK7N27Fxs3bsTq1aula86YMQMJCQkYOHAgBg8ejPT0dPzrX/9CRkZGveNiEkBERPLSBDsGJiQk4MaNG5g7dy4KCwsRGhqK9PR0abJgXl4elMrqzvny8nJMmTIFBQUFcHZ2RnBwMDZt2oSEhASpzqhRo5CamgqtVouXXnoJ3bp1w8cff4wBAwbUOy7uE0BERE2uUfcJWLrI8n0CZtg21sbCngAiIpKVhu76d297e8EkgIiI5IVPEZRwdQAREZFMMQkgIiKSKQ4HEBGRrChg4ZwAq0XS9JgEEBGRvDTBEsHmisMBREREMsWeACIikheuDpAwCSAiInlhEiDhcAAREZFMsSeAiIhkhTsGVjO7J+DQoUN44okn4OvrC4VCgV27dhmdF0Jg7ty58PHxgbOzM6Kjo3HhwoX7XjclJQX+/v5Qq9WIiIjAsWPHzA2NiIjo/oQVip0wOwkoLy9HSEgIUlJSTJ7/29/+hvfeew+pqan46quv4OLigpiYGNy5c6fWa27fvh0ajQbz5s3DyZMnERISgpiYGFy/ft3c8IiIiKiezE4C4uLisGjRIowaNarGOSEEVqxYgddffx0jRoxAnz598OGHH+Lq1as1egzutWzZMkycOBHJycno0aMHUlNT0bJlS6xdu9bc8IiIiOrGngCJVScG5ubmorCwENHR0dIxd3d3REREICsry2SbyspKnDhxwqiNUqlEdHR0rW0qKipQWlpqVIiIiOqjak6AJcVeWDUJKCwsBAB4eXkZHffy8pLO/VZxcTH0er1ZbbRaLdzd3aXi5+dnheiJiIjk5YFcIjh79myUlJRIJT8/v6lDIiKiB0XVtsGWFDth1SWC3t7eAICioiL4+PhIx4uKihAaGmqyjaenJxwcHFBUVGR0vKioSLreb6lUKqhUKusETURE8sLNgiRW7Qno3LkzvL29odPppGOlpaX46quvEBkZabKNk5MTwsLCjNoYDAbodLpa2xARETUU5wRUM7snoKysDBcvXpRe5+bmIjs7G23atEHHjh3x8ssvY9GiRejSpQs6d+6MN954A76+vhg5cqTUZsiQIRg1ahSmTZsGANBoNEhKSkJ4eDj69euHFStWoLy8HMnJyZZ/QiIiIjLJ7CTg+PHjGDx4sPRao9EAAJKSkrB+/Xq89tprKC8vx6RJk3Dr1i0MGDAA6enpUKvVUptLly6huLhYep2QkIAbN25g7ty5KCwsRGhoKNLT02tMFiQiIrIYhwMkCiHEA/9xSktL4e7ujo5vLYLynmSDiIgeDIY7d5A363WUlJTAzc3NJu9R9V0R8Mb/wcGC7wr9nTv4fuFfbRprY3kgVwcQERGR5fgAISIikhcOB0iYBBARkbwwCZBwOICIiEim2BNARESyYulaf3vaJ4A9AURERDLFJICIiEimOBxARETywomBEiYBREQkK5wTUI1JABERyY8dfZFbgnMCiIiIZIo9AUREJC+cEyBhEkBERLLCOQHVOBxAREQkU+wJICIieeFwgIRJABERyQqHA6pxOICIiEim2BNARETywuEACZMAIiKSFyYBEg4HEBERNYKUlBT4+/tDrVYjIiICx44dq7VuWloawsPD4eHhARcXF4SGhmLjxo1GdcaPHw+FQmFUYmNjzYqJPQFERCQrTTExcPv27dBoNEhNTUVERARWrFiBmJgY5OTkoH379jXqt2nTBnPmzEFwcDCcnJywZ88eJCcno3379oiJiZHqxcbGYt26ddJrlUplVlzsCSAiInkRVihmWrZsGSZOnIjk5GT06NEDqampaNmyJdauXWuyflRUFEaNGoXu3bsjMDAQ06dPR58+fXDkyBGjeiqVCt7e3lJp3bq1WXExCSAiInmxUhJQWlpqVCoqKky+XWVlJU6cOIHo6GjpmFKpRHR0NLKysu4frhDQ6XTIycnBwIEDjc5lZGSgffv26NatGyZPnoybN2/W/z6ASQAREVGD+Pn5wd3dXSpardZkveLiYuj1enh5eRkd9/LyQmFhYa3XLykpQatWreDk5IT4+HisXLkSQ4cOlc7Hxsbiww8/hE6nw5IlS3Dw4EHExcVBr9fX+zNwTgAREcmKteYE5Ofnw83NTTpu7nj8/bi6uiI7OxtlZWXQ6XTQaDQICAhAVFQUAODpp5+W6vbu3Rt9+vRBYGAgMjIyMGTIkHq9B5MAIiKSFystEXRzczNKAmrj6ekJBwcHFBUVGR0vKiqCt7d3re2USiWCgoIAAKGhoTh37hy0Wq2UBPxWQEAAPD09cfHixXonARwOICIisiEnJyeEhYVBp9NJxwwGA3Q6HSIjI+t9HYPBUOu8AwAoKCjAzZs34ePjU+9rsieAiIhkpSmWCGo0GiQlJSE8PBz9+vXDihUrUF5ejuTkZABAYmIiOnToIM0r0Gq1CA8PR2BgICoqKrB3715s3LgRq1evBgCUlZVhwYIFeOqpp+Dt7Y1Lly7htddeQ1BQkNESwvthEkBERPLSBDsGJiQk4MaNG5g7dy4KCwsRGhqK9PR0abJgXl4elMrqzvny8nJMmTIFBQUFcHZ2RnBwMDZt2oSEhAQAgIODA06fPo0NGzbg1q1b8PX1xbBhw7Bw4UKz5iYohBAP/AaIpaWlcHd3R8e3FkGpVjd1OEREZCbDnTvIm/U6SkpK6jXO3hBV3xXdp/4fHFQN/67QV9zBuZS/2jTWxsKeACIikhc+O0DCJICIiGRF8b9iSXt7wdUBREREMsWeACIikhcOB0iYBBARkaw0xRLB5srs4YBDhw7hiSeegK+vLxQKBXbt2iWdu3v3LmbOnInevXvDxcUFvr6+SExMxNWrV+u85vz582s8Ezk4ONjsD0NERHRfTfAUwebK7CSgvLwcISEhSElJqXHu559/xsmTJ/HGG2/g5MmTSEtLQ05ODoYPH37f6/bs2RPXrl2Tym8fl0hERETWZfZwQFxcHOLi4kyec3d3x759+4yOrVq1Cv369UNeXh46duxYeyAtWtS5hzIREZHV2NFf85aw+eqAkpISKBQKeHh41FnvwoUL8PX1RUBAAMaNG4e8vLxa61ZUVNR4jjMREVF9VM0JsKTYC5smAXfu3MHMmTMxduzYOndVioiIwPr165Geno7Vq1cjNzcXjz32GG7fvm2yvlarNXqGs5+fn60+AhERkd2yWRJw9+5djBkzBkII6YEHtYmLi8Po0aPRp08fxMTEYO/evbh16xY++ugjk/Vnz56NkpISqeTn59viIxARkT3ixECJTZYIViUAV65cwRdffGH23soeHh7o2rUrLl68aPK8SqUy6wEJREREVbhEsJrVewKqEoALFy5g//79aNu2rdnXKCsrw6VLl8x6JjIRERGZx+wkoKysDNnZ2cjOzgYA5ObmIjs7G3l5ebh79y7++Mc/4vjx49i8eTP0ej0KCwtRWFiIyspK6RpDhgzBqlWrpNevvvoqDh48iMuXLyMzMxOjRo2Cg4MDxo4da/knJCIiuheHAyRmDwccP34cgwcPll5rNBoAQFJSEubPn49PPvkEABAaGmrU7sCBA4iKigIAXLp0CcXFxdK5goICjB07Fjdv3kS7du0wYMAAHD16FO3atTM3PCIiojpxOKCa2UlAVFQUhKj9DtR1rsrly5eNXm/bts3cMIiIiMhCfHYAERHJCx8gJGESQERE8sIkQMIkgIiIZIVzAqrZfNtgIiIiap7YE0BERPLC4QAJkwAiIpIVhRBQ1GMlW13t7QWHA4iIiGSKPQFERCQvHA6QMAkgIiJZ4eqAahwOICIikin2BBARkbxwOEDCJICIiGSFwwHVOBxAREQkU+wJICIieeFwgIRJABERyQqHA6oxCSAiInlhT4CEcwKIiIhkij0BREQkO/bUpW8JJgFERCQvQvxaLGlvJzgcQEREJFPsCSAiIlnh6oBqTAKIiEheuDpAwuEAIiIimWJPABERyYrC8GuxpL29YBJARETywuEACYcDiIiIGkFKSgr8/f2hVqsRERGBY8eO1Vo3LS0N4eHh8PDwgIuLC0JDQ7Fx48Za67/wwgtQKBRYsWKFWTExCSAiIlmpWh1gSTHX9u3bodFoMG/ePJw8eRIhISGIiYnB9evXTdZv06YN5syZg6ysLJw+fRrJyclITk7GZ599VqPuzp07cfToUfj6+podF5MAIiKSl6rNgiwpAEpLS41KRUVFrW+5bNkyTJw4EcnJyejRowdSU1PRsmVLrF271mT9qKgojBo1Ct27d0dgYCCmT5+OPn364MiRI0b1fvjhB7z44ovYvHkzHB0dzb4VTAKIiEhWrNUT4OfnB3d3d6lotVqT71dZWYkTJ04gOjpaOqZUKhEdHY2srKz7xiuEgE6nQ05ODgYOHCgdNxgMeOaZZzBjxgz07NmzQfeCEwOJiIgaID8/H25ubtJrlUplsl5xcTH0ej28vLyMjnt5eeG7776r9folJSXo0KEDKioq4ODggPfffx9Dhw6Vzi9ZsgQtWrTASy+91ODPwCSAiIjkxUqrA9zc3IySAGtzdXVFdnY2ysrKoNPpoNFoEBAQgKioKJw4cQLvvvsuTp48CYVC0eD3YBJARESy0tjbBnt6esLBwQFFRUVGx4uKiuDt7V1rO6VSiaCgIABAaGgozp07B61Wi6ioKBw+fBjXr19Hx44dpfp6vR5/+ctfsGLFCly+fLlesXFOABERkQ05OTkhLCwMOp1OOmYwGKDT6RAZGVnv6xgMBmny4TPPPIPTp08jOztbKr6+vpgxY4bJFQS1YU8AERHJSxM8Slij0SApKQnh4eHo168fVqxYgfLyciQnJwMAEhMT0aFDB2lyoVarRXh4OAIDA1FRUYG9e/di48aNWL16NQCgbdu2aNu2rdF7ODo6wtvbG926dat3XEwCiIhIVpriKYIJCQm4ceMG5s6di8LCQoSGhiI9PV2aLJiXlwelsrpzvry8HFOmTEFBQQGcnZ0RHByMTZs2ISEhoeGBm2D2cMChQ4fwxBNPwNfXFwqFArt27TI6P378eCgUCqMSGxt73+uas5MSERHRg2batGm4cuUKKioq8NVXXyEiIkI6l5GRgfXr10uvFy1ahAsXLuCXX37Bjz/+iMzMzPsmAJcvX8bLL79sVkxmJwHl5eUICQlBSkpKrXViY2Nx7do1qWzdurXOa5q7kxIREVGDCSsUO2H2cEBcXBzi4uLqrKNSqeqc8fhb9+6kBACpqan49NNPsXbtWsyaNcvcEImIiGrVFMMBzZVNVgdkZGSgffv26NatGyZPnoybN2/WWrchOylVVFTU2K6RiIiIzGP1JCA2NhYffvghdDodlixZgoMHDyIuLg56vd5k/bp2UiosLDTZRqvVGm3V6OfnZ+2PQURE9sogLC92wuqrA55++mnpv3v37o0+ffogMDAQGRkZGDJkiFXeY/bs2dBoNNLr0tJSJgJERFQ/Vtox0B7YfLOggIAAeHp64uLFiybPN2QnJZVKJW3XaOttG4mIyL4oYOEDhJr6A1iRzZOAgoIC3Lx5Ez4+PibPW2snJSIiIjKP2UlAWVmZtEUhAOTm5iI7Oxt5eXkoKyvDjBkzcPToUVy+fBk6nQ4jRoxAUFAQYmJipGsMGTIEq1atkl5rNBr84x//wIYNG3Du3DlMnjzZaCclIiIiq6naMdCSYifMnhNw/PhxDB48WHpdNTaflJSE1atX4/Tp09iwYQNu3boFX19fDBs2DAsXLjR6xOKlS5dQXFwsvb7fTkpERETWwiWC1cxOAqKioiDqyILq8+ACU083mjZtGqZNm2ZuOERERNRAfHYAERHJC1cHSJgEEBGRrCiEgMKCcX1L2jY3Nl8dQERERM0TewKIiEheDP8rlrS3E0wCiIhIVjgcUI3DAURERDLFngAiIpIXrg6QMAkgIiJ5sXTXPzsaDmASQEREssIdA6txTgAREZFMsSeAiIjkhcMBEiYBREQkKwrDr8WS9vaCwwFEREQyxZ4AIiKSFw4HSJgEEBGRvHCfAAmHA4iIiGSKPQFERCQrfHZANSYBREQkL5wTIOFwABERkUyxJ4CIiORFALBkrb/9dAQwCSAiInnhnIBqTAKIiEheBCycE2C1SJoc5wQQERHJFHsCiIhIXrg6QMIkgIiI5MUAQGFhezvB4QAiIiKZYk8AERHJClcHVGMSQERE8sI5ARIOBxAREckUewKIiEhe2BMgYU8AERHJS1USYElpgJSUFPj7+0OtViMiIgLHjh2rtW5aWhrCw8Ph4eEBFxcXhIaGYuPGjUZ15s+fj+DgYLi4uKB169aIjo7GV199ZVZMTAKIiIhsbPv27dBoNJg3bx5OnjyJkJAQxMTE4Pr16ybrt2nTBnPmzEFWVhZOnz6N5ORkJCcn47PPPpPqdO3aFatWrcKZM2dw5MgR+Pv7Y9iwYbhx40a941II8eD3a5SWlsLd3R0d31oEpVrd1OEQEZGZDHfuIG/W6ygpKYGbm5tN3qPqu2JIt7+ghYOqwdf5r74Cupx3zIo1IiICffv2xapVqwAABoMBfn5+ePHFFzFr1qx6XeORRx5BfHw8Fi5caPJ81efbv38/hgwZUq9rsieAiIhkpWqJoCUF+PVL995SUVFh8v0qKytx4sQJREdHS8eUSiWio6ORlZV133iFENDpdMjJycHAgQNrfY+///3vcHd3R0hISL3vBZMAIiKSFyvNCfDz84O7u7tUtFqtybcrLi6GXq+Hl5eX0XEvLy8UFhbWGmZJSQlatWoFJycnxMfHY+XKlRg6dKhRnT179qBVq1ZQq9VYvnw59u3bB09Pz3rfCrOTgEOHDuGJJ56Ar68vFAoFdu3aZXReoVCYLEuXLq31mvPnz69RPzg42NzQiIiIGk1+fj5KSkqkMnv2bKte39XVFdnZ2fjPf/6DxYsXQ6PRICMjw6jO4MGDkZ2djczMTMTGxmLMmDG1zjMwxewlguXl5QgJCcGzzz6LJ598ssb5a9euGb3+97//jQkTJuCpp56q87o9e/bE/v37qwNrwdWLRERkAwYBKCyYDmf4ta2bm1u95gR4enrCwcEBRUVFRseLiorg7e1dazulUomgoCAAQGhoKM6dOwetVouoqCipjouLC4KCghAUFITf/e536NKlC9asWVPvhMTsb9q4uDjExcXVev63H2j37t0YPHgwAgIC6g6kRYs6bwYREZFVNPI+AU5OTggLC4NOp8PIkSMB/DoxUKfTYdq0afW+jsFgqHXegTl17mXTP7eLiorw6aefYsOGDfete+HCBfj6+kKtViMyMhJarRYdO3Y0WbeiosLoQ5aWllotZiIiImvTaDRISkpCeHg4+vXrhxUrVqC8vBzJyckAgMTERHTo0EGaV6DVahEeHo7AwEBUVFRg79692LhxI1avXg3g1175xYsXY/jw4fDx8UFxcTFSUlLwww8/YPTo0fWOy6ZJwIYNG+Dq6mpy2OBeERERWL9+Pbp164Zr165hwYIFeOyxx3D27Fm4urrWqK/VarFgwQJbhU1ERHbNwp4AmN82ISEBN27cwNy5c1FYWIjQ0FCkp6dLkwXz8vKgVFZP0ysvL8eUKVNQUFAAZ2dnBAcHY9OmTUhISAAAODg44LvvvsOGDRtQXFyMtm3bom/fvjh8+DB69uxZ77gs2idAoVBg586dUvfGbwUHB2Po0KFYuXKlWde9desWOnXqhGXLlmHChAk1zpvqCfDz8+M+AURED6jG3CcguvOLaKG0YJ8AQwX25660aayNxWY9AYcPH0ZOTg62b99udlsPDw907doVFy9eNHlepVJBpWr4PyARERHZcJ+ANWvWICwszKxNC6qUlZXh0qVL8PHxsUFkREQkawZhebETZicBZWVlyM7ORnZ2NgAgNzcX2dnZyMvLk+qUlpZix44deO6550xeY8iQIdLWiQDw6quv4uDBg7h8+TIyMzMxatQoODg4YOzYseaGR0REVDdhsLzYCbOHA44fP47BgwdLrzUaDQAgKSkJ69evBwBs27YNQohav8QvXbqE4uJi6XVBQQHGjh2Lmzdvol27dhgwYACOHj2Kdu3amRseERER1ZPZSUBUVBTuN5dw0qRJmDRpUq3nL1++bPR627Zt5oZBRETUMI28T0Bzxm35iIhIXgwCDVnmZ9zePjAJICIieWFPgIRPESQiIpIp9gQQEZG8CFjYE2C1SJockwAiIpIXDgdIOBxAREQkU+wJICIieTEYAFiw4Y9BxpsFERERPdA4HCDhcAAREZFMsSeAiIjkhT0BEiYBREQkL9wxUMLhACIiIpliTwAREcmKEAYICx4HbEnb5oZJABERyYsQlnXpc04AERHRA0pYOCfAjpIAzgkgIiKSKfYEEBGRvBgMgMKCcX3OCSAiInpAcThAwuEAIiIimWJPABERyYowGCAsGA7gEkEiIqIHFYcDJBwOICIikin2BBARkbwYBKBgTwDAJICIiORGCACWLBG0nySAwwFEREQyxZ4AIiKSFWEQEBYMBwg76glgEkBERPIiDLBsOIBLBImIiB5I7AmoxjkBREREMmUXPQFVWZnhzp0mjoSIiBqi6v+/G+Ov7P+KCou69P+Lu1aMpmkphB30axQUFMDPz6+pwyAiIgvl5+fjoYcessm179y5g86dO6OwsNDia3l7eyM3NxdqtdoKkTUdu0gCDAYDrl69CldXVygUilrrlZaWws/PD/n5+XBzc2vECC3DuBvXgxo38ODGzrgbV3OMWwiB27dvw9fXF0ql7Uaq79y5g8rKSouv4+Tk9MAnAICdDAcolUqzMkc3N7dm84NvDsbduB7UuIEHN3bG3biaW9zu7u42fw+1Wm0XX97WwomBREREMsUkgIiISKZklQSoVCrMmzcPKpWqqUMxC+NuXA9q3MCDGzvjblwPatxkfXYxMZCIiIjMJ6ueACIiIqrGJICIiEimmAQQERHJFJMAIiIimWISQEREJFN2lwSkpKTA398farUaEREROHbsWJ31d+zYgeDgYKjVavTu3Rt79+5tpEh/pdVq0bdvX7i6uqJ9+/YYOXIkcnJy6myzfv16KBQKo9LYO2DNnz+/RgzBwcF1tmnqew0A/v7+NeJWKBSYOnWqyfpNea8PHTqEJ554Ar6+vlAoFNi1a5fReSEE5s6dCx8fHzg7OyM6OhoXLly473XN/R2xZtx3797FzJkz0bt3b7i4uMDX1xeJiYm4evVqnddsyM+bNeMGgPHjx9eIITY29r7Xbcr7DcDkz7tCocDSpUtrvWZj3G9qHuwqCdi+fTs0Gg3mzZuHkydPIiQkBDExMbh+/brJ+pmZmRg7diwmTJiAU6dOYeTIkRg5ciTOnj3baDEfPHgQU6dOxdGjR7Fv3z7cvXsXw4YNQ3l5eZ3t3NzccO3aNalcuXKlkSKu1rNnT6MYjhw5Umvd5nCvAeA///mPUcz79u0DAIwePbrWNk11r8vLyxESEoKUlBST5//2t7/hvffeQ2pqKr766iu4uLggJiYGd+p4mqa5vyPWjvvnn3/GyZMn8cYbb+DkyZNIS0tDTk4Ohg8fft/rmvPzZu24q8TGxhrFsHXr1jqv2dT3G4BRvNeuXcPatWuhUCjw1FNP1XldW99vaiaEHenXr5+YOnWq9Fqv1wtfX1+h1WpN1h8zZoyIj483OhYRESGef/55m8ZZl+vXrwsA4uDBg7XWWbdunXB3d2+8oEyYN2+eCAkJqXf95nivhRBi+vTpIjAwUBgMBpPnm8O9FkIIAGLnzp3Sa4PBILy9vcXSpUulY7du3RIqlUps3bq11uuY+zti7bhNOXbsmAAgrly5Umsdc3/eLGUq7qSkJDFixAizrtMc7/eIESPE448/Xmedxr7f1HTspiegsrISJ06cQHR0tHRMqVQiOjoaWVlZJttkZWUZ1QeAmJiYWus3hpKSEgBAmzZt6qxXVlaGTp06wc/PDyNGjMA333zTGOEZuXDhAnx9fREQEIBx48YhLy+v1rrN8V5XVlZi06ZNePbZZ+t8+mRzuNe/lZubi8LCQqN76u7ujoiIiFrvaUN+RxpDSUkJFAoFPDw86qxnzs+brWRkZKB9+/bo1q0bJk+ejJs3b9Zatzne76KiInz66aeYMGHCfes2h/tNtmc3SUBxcTH0ej28vLyMjnt5edX67OjCwkKz6tuawWDAyy+/jEcffRS9evWqtV63bt2wdu1a7N69G5s2bYLBYED//v1RUFDQaLFGRERg/fr1SE9Px+rVq5Gbm4vHHnsMt2/fNlm/ud1rANi1axdu3bqF8ePH11qnOdxrU6rumzn3tCG/I7Z2584dzJw5E2PHjq3zaXbm/rzZQmxsLD788EPodDosWbIEBw8eRFxcHPR6vcn6zfF+b9iwAa6urnjyySfrrNcc7jc1Drt4lLC9mDp1Ks6ePXvfsbfIyEhERkZKr/v374/u3bvjgw8+wMKFC20dJgAgLi5O+u8+ffogIiICnTp1wkcffVSvvzKagzVr1iAuLg6+vr611mkO99pe3b17F2PGjIEQAqtXr66zbnP4eXv66ael/+7duzf69OmDwMBAZGRkYMiQIY0Sg6XWrl2LcePG3Xdya3O439Q47KYnwNPTEw4ODigqKjI6XlRUBG9vb5NtvL29zapvS9OmTcOePXtw4MABPPTQQ2a1dXR0xMMPP4yLFy/aKLr78/DwQNeuXWuNoTndawC4cuUK9u/fj+eee86sds3hXgOQ7ps597QhvyO2UpUAXLlyBfv27TP7mfb3+3lrDAEBAfD09Kw1huZ0vwHg8OHDyMnJMftnHmge95tsw26SACcnJ4SFhUGn00nHDAYDdDqd0V9y94qMjDSqDwD79u2rtb4tCCEwbdo07Ny5E1988QU6d+5s9jX0ej3OnDkDHx8fG0RYP2VlZbh06VKtMTSHe32vdevWoX379oiPjzerXXO41wDQuXNneHt7G93T0tJSfPXVV7Xe04b8jthCVQJw4cIF7N+/H23btjX7Gvf7eWsMBQUFuHnzZq0xNJf7XWXNmjUICwtDSEiI2W2bw/0mG2nqmYnWtG3bNqFSqcT69evFt99+KyZNmiQ8PDxEYWGhEEKIZ555RsyaNUuq/+WXX4oWLVqIt99+W5w7d07MmzdPODo6ijNnzjRazJMnTxbu7u4iIyNDXLt2TSo///yzVOe3cS9YsEB89tln4tKlS+LEiRPi6aefFmq1WnzzzTeNFvdf/vIXkZGRIXJzc8WXX34poqOjhaenp7h+/brJmJvDva6i1+tFx44dxcyZM2uca073+vbt2+LUqVPi1KlTAoBYtmyZOHXqlDSL/q233hIeHh5i9+7d4vTp02LEiBGic+fO4pdffpGu8fjjj4uVK1dKr+/3O2LruCsrK8Xw4cPFQw89JLKzs41+5isqKmqN+34/b7aO+/bt2+LVV18VWVlZIjc3V+zfv1888sgjokuXLuLOnTu1xt3U97tKSUmJaNmypVi9erXJazTF/abmwa6SACGEWLlypejYsaNwcnIS/fr1E0ePHpXODRo0SCQlJRnV/+ijj0TXrl2Fk5OT6Nmzp/j0008bNV4AJsu6detqjfvll1+WPqOXl5f4/e9/L06ePNmocSckJAgfHx/h5OQkOnToIBISEsTFixdrjVmIpr/XVT777DMBQOTk5NQ415zu9YEDB0z+bFTFZzAYxBtvvCG8vLyESqUSQ4YMqfGZOnXqJObNm2d0rK7fEVvHnZubW+vP/IEDB2qN+34/b7aO++effxbDhg0T7dq1E46OjqJTp05i4sSJNb7Mm9v9rvLBBx8IZ2dncevWLZPXaIr7Tc2DQgghbNrVQERERM2S3cwJICIiIvMwCSAiIpIpJgFEREQyxSSAiIhIppgEEBERyRSTACIiIpliEkBERCRTTAKIiIhkikkAERGRTDEJICIikikmAURERDL1/6uOdZlrHMigAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAGzCAYAAABO7D91AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASIZJREFUeJzt3XtcVGX+B/DPcJvxAnhBGTDwFoIIQmKykP28UUCsgbVqrKtIpulK2VJkmIllLZnlJTWpLS+liLEptuZSiuIlMBPQ0krFUPAyoJaAmKAzz+8Pl8mJGeQ4Mwiez7vX83o153yfM985QfPleZ5zjkIIIUBERERkhM2dToCIiIhaLhYKREREZBILBSIiIjKJhQIRERGZxEKBiIiITGKhQERERCaxUCAiIiKTWCgQERGRSSwUiIiIyCQWCiQLEydORI8ePe50Gs1i6NChGDp06B17f4VCgYSEhDv2/kRkWSwUyKpKSkqQkJCAPn36oG3btmjbti18fX0xffp0fPfdd3c6PZOqqqrw6quvIiAgAO3bt0ebNm3g5+eHmTNn4uzZs0b7jBkzBgqFAjNnzjS6Pzc3FwqFAgqFAmvXrjUa88ADD0ChUMDPz89in8Ua8vLyMHfuXFy6dKnZ3/vcuXN46aWXMGzYMDg6OkKhUCA3N9dkfF1dHf75z3/Cx8cHKpUKrq6uiIqKwunTp5svaaJWjIUCWc2WLVvg5+eHTz75BGFhYVi0aBGWLFmCyMhIbN26FYGBgTh16tSdTrOBn3/+GYGBgZg3bx58fX0xf/58vPvuuxg2bBg++ugjo3+tV1VV4T//+Q969OiB9evXo7FHqKhUKqSnpzfYfvLkSeTl5UGlUlny41hFXl4eXn311TtSKBw9ehTz58/HmTNn4O/v32jstWvXEBUVhTfeeAMRERF477338OKLL6Jdu3aorKxspoyJWje7O50A3Z1OnDiBJ554At27d0dOTg7c3NwM9s+fPx/vvfcebGwar1VramrQrl07a6Zq4Pr163jsscdQXl6O3NxcDB482GD/G2+8gfnz5zfo99lnn0Gr1WLlypUYPnw4du/ejSFDhhh9j0ceeQSff/45Lly4ABcXF/329PR0uLq6wsvLC7/++qtlP9hdJCgoCBcvXkSnTp3w73//G6NHjzYZu2jRIuzatQt79+7FoEGDmjFLorsHRxTIKt566y3U1NRg1apVDYoEALCzs8Ozzz4LDw8P/baJEyeiffv2OHHiBB555BE4Ojpi3LhxAIA9e/Zg9OjR8PT0hFKphIeHB/7xj3/gt99+a3DsrKws+Pn5QaVSwc/PD5s2bWpy3p999hkOHTqEl19+uUGRAABOTk544403Gmxft24dHnroIQwbNgx9+/bFunXrTL5HdHQ0lEolMjMzDbanp6djzJgxsLW1bXK+H3zwAXr37o02bdpg0KBB2LNnj9G42tpapKSk4N5779WfvxdffBG1tbUGcfXrC9atWwdvb2+oVCoEBQVh9+7d+pi5c+ciKSkJANCzZ0/9dMrJkycNjlX/30GpVKJfv37Izs5u8udqjKOjIzp16nTLOJ1OhyVLlmDUqFEYNGgQrl+/jitXrlgkByI54YgCWcWWLVtw7733Ijg4WFK/69evIzw8HIMHD8bbb7+Ntm3bAgAyMzNx5coVTJs2DZ07d8b+/fuxdOlSnD592uAL96uvvsLjjz8OX19fpKam4uLFi4iPj8c999zTpPf//PPPAQDjx49vcs5nz57Fzp07sWbNGgBAbGwsFi1ahGXLlsHBwaFBfNu2bREdHY3169dj2rRpAIBDhw7hyJEj+PDDD5u8duOjjz7C008/jdDQUDz33HP4+eef8eijj6JTp04GBZhOp8Ojjz6KvXv3YsqUKejbty++//57LFq0CMeOHUNWVpbBcXft2oUNGzbg2WefhVKpxHvvvYeIiAjs378ffn5+eOyxx3Ds2DGsX78eixYt0o+KdOnSRX+MvXv3YuPGjfj73/8OR0dHvPvuu3j88cdRWlqKzp07A7gxLdDU4f9OnTrdcvTpj3744QecPXsW/fv3x5QpU7BmzRrU1dXB398fS5YswbBhwyQdj0i2BJGFVVZWCgAiJiamwb5ff/1VnD9/Xt+uXLmi3xcXFycAiJdeeqlBv5vj6qWmpgqFQiFOnTql3xYYGCjc3NzEpUuX9Nu++uorAUB07979lrnfd999wtnZ+ZZxN3v77bdFmzZtRFVVlRBCiGPHjgkAYtOmTQZxO3fuFABEZmam2LJli1AoFKK0tFQIIURSUpLo1auXEEKIIUOGiH79+jX6nnV1daJr164iMDBQ1NbW6rd/8MEHAoAYMmSIftsnn3wibGxsxJ49ewyOkZaWJgCIr7/+Wr8NgAAgDhw4oN926tQpoVKpxKhRo/TbFixYIACIkpKSBrkBEA4ODqK4uFi/7dChQwKAWLp0aYPz0ZRm7H2EECIzM1MAEDt37mywb+PGjQKA6Ny5s/Dy8hKrVq0Sq1atEl5eXsLBwUEcOnTI6DGJyBBHFMjiqqqqAADt27dvsG/o0KE4dOiQ/vWCBQvwwgsvGMTU/5V9szZt2uj/vaamBr/99htCQ0MhhEBRURE8PT1x7tw5HDx4EC+99BKcnZ318Q899BB8fX1RU1PTpNwdHR1v/SFvsm7dOkRFRen7eXl5ISgoCOvWrUNMTIzRPg8//DA6deqEjIwMvPDCC8jIyMCECROa/J4HDhxARUUFXnvtNYNRi4kTJ+qnBeplZmaib9++8PHxwYULF/Tbhw8fDgDYuXMnQkND9dtDQkIQFBSkf+3p6Yno6Gj85z//gVarbdLUSFhYGHr37q1/3b9/fzg5OeHnn3/WbwsICMC2bdua9HnVanWT4m52+fJlAEB1dTWKior0oyzDhw/Hvffei7feesvk1SdE9DsWCmRx9V+Y9f+jvtn777+P6upqlJeX429/+1uD/XZ2dkanCUpLSzFnzhx8/vnnDRb61Q9f119B4eXl1aC/t7c3CgsL9a/Pnz8PrVarf92+fXu0b9++wZfZrfz4448oKirChAkTUFxcrN8+dOhQLF++HFVVVXBycmrQz97eHqNHj0Z6ejoGDRqEsrIy/PWvf23y+5r6rPb29ujVq5fBtuPHj+PHH380mBq4WUVFhcFrY+evT58+uHLlCs6fP9+kL21PT88G2zp27Gjw365jx44ICwu75bFuV31x+cADDxhMxXh6emLw4MHIy8uz2nsT3U1YKJDFOTs7w83NDYcPH26wr37Nwh8XvtVTKpUN5qK1Wi0eeugh/PLLL5g5cyZ8fHzQrl07nDlzBhMnToROp5Oc4/33329waWZKSgrmzp0LHx8fFBUVoayszODLxZT6v0j/8Y9/4B//+EeD/Z999hni4+ON9v3rX/+KtLQ0zJ07FwEBAfD19ZX8OZpCp9PB398fCxcuNLq/KZ9TKlOjDuKmy0br6urwyy+/NOl4Xbp0kbTIEwDc3d0BAK6urg32de3aFUVFRZKORyRXLBTIKqKiovDhhx9i//79Zl+W9v333+PYsWNYs2aNwfD8H4etu3fvDuDGX9B/dPToUYPX69atM7hiov6v8JEjR2L9+vVYu3YtkpOTG81LCIH09HQMGzYMf//73xvsnzdvHtatW2eyUBg8eDA8PT2Rm5tr9JLLxtz8WeunEIAbCwRLSkoQEBCg39a7d28cOnQII0aMgEKhuOWxjZ2/Y8eOoW3btvpRiaYc51by8vKavKCwpKRE8p01/f39YW9vjzNnzjTYd/bsWZMjLERkiIUCWcWLL76I9PR0PPnkk8jJyWnwV51o5IZEf1T/l+TNfYQQWLJkiUGcm5sbAgMDsWbNGoN1Ctu2bcMPP/yg/3IFbgxHG/OXv/wFqampeOONNzB06FCEhIQY7K+ursabb76JN954A19//TVOnjyJ1157DX/5y18aHOvYsWN45ZVXcPbsWf1ftzdTKBR49913UVRUJOkqCwAYOHAgunTpgrS0NMTHx+vXKaxevbrBTZDGjBmDrVu34l//+hemTJlisO+3336DTqczuFdFfn4+CgsLMWDAAABAWVkZNm/ejIiICP1/i/p4c264ZO01Co6OjnjkkUewZcsW/PTTT/Dx8QFwY7ooLy8PTz/9tORjEskRCwWyCi8vL6SnpyM2Nhbe3t4YN24cAgICIIRASUkJ0tPTYWNj06TLFn18fNC7d2+88MILOHPmDJycnPDZZ58ZvSlRamoqoqKiMHjwYDz55JP45ZdfsHTpUvTr18/omok/sre3x8aNGxEWFob/+7//w5gxY/DAAw/A3t4eR44cQXp6Ojp27Ig33ngD69atg62tLaKioowe69FHH8XLL7+MjIwMJCYmGo2Jjo5GdHT0LfMylufrr7+Op59+GsOHD8fYsWNRUlKCVatWNVijMH78eHz66aeYOnUqdu7ciQceeABarRY//fQTPv30U3z55ZcYOHCgPt7Pzw/h4eEGl0cCwKuvvqqPqV/s+PLLL+OJJ56Avb09Ro4cKenmWOasUXj99dcBAEeOHAEAfPLJJ9i7dy8AYPbs2fq4f/7zn8jJycHw4cPx7LPPAgDeffdddOrUCbNmzbqt9yaSnTt4xQXJQHFxsZg2bZq49957hUqlEm3atBE+Pj5i6tSp4uDBgwaxcXFxol27dkaP88MPP4iwsDDRvn174eLiIiZPnqy/5G7VqlUGsZ999pno27evUCqVwtfXV2zcuFHExcU16fLIer/++quYM2eO8Pf3F23bthUqlUr4+fmJ5ORkce7cOVFXVyc6d+4sHnzwwUaP07NnT3HfffcJIQwvj2xMUy6PrPfee++Jnj17CqVSKQYOHCh2794thgwZYnB5pBA3LqecP3++6Nevn1AqlaJjx44iKChIvPrqq6KyslIfB0BMnz5drF27Vnh5eQmlUinuu+8+o5cfzps3T3Tr1k3Y2NgYXMJYf4w/6t69u4iLi2vS57oVNHIp5R8VFBSIsLAw0a5dO+Ho6Ciio6PFsWPHLJIHkRwohJAwBkxEdzWFQoHp06dj2bJldzoVImoheAtnIiIiMomFAhEREZnEQoGIiIhM4lUPRKTHJUtE9EccUSAiIiKTWCgQERGRSXfF1INOp8PZs2fh6OhokVvLEhFR8xJCoLq6Gu7u7g2e92JJV69eRV1dndnHcXBwgEqlskBGLd9dUSicPXvWKg+2ISKi5lVWVtakO7bejqtXr6Jn9/bQVGhvHXwLarUaJSUlsigW7opCof6xxvekzIaNDP6jERHdbXRXr+L0q6/r/39uDXV1ddBUaFFS0B1Ojrc/alFVrUPPoFOoq6tjodBa1E832KhULBSIiFqx5pg+dnK0MatQkJu7olAgIiJqKq3QQWvGlcBaobNcMq0ACwUiIpIVHQR0uP1KwZy+rZHVxl6WL1+OHj16QKVSITg4GPv37280PjMzEz4+PlCpVPD398fWrVutlRoREcmYzgL/yIlVCoUNGzYgMTERKSkpKCwsREBAAMLDw1FRUWE0Pi8vD7GxsZg0aRKKiooQExODmJgYHD582BrpERERURNZpVBYuHAhJk+ejPj4ePj6+iItLQ1t27bFypUrjcYvWbIEERERSEpKQt++fTFv3jwMGDCAj7olIiKL0wphdpMTixcKdXV1KCgoQFhY2O9vYmODsLAw5OfnG+2Tn59vEA8A4eHhJuNra2tRVVVl0IiIiJqifo2COU1OLF4oXLhwAVqtFq6urgbbXV1dodFojPbRaDSS4lNTU+Hs7KxvvNkSERGRdbTKC0mTk5NRWVmpb2VlZXc6JSIiaiV0ENCa0eQ2omDxyyNdXFxga2uL8vJyg+3l5eVQq9VG+6jVaknxSqUSSqXSMgkTEZGs8PJIaSw+ouDg4ICgoCDk5OTot+l0OuTk5CAkJMRon5CQEIN4ANi2bZvJeCIiImoeVrnhUmJiIuLi4jBw4EAMGjQIixcvRk1NDeLj4wEAEyZMQLdu3ZCamgoAmDFjBoYMGYJ33nkHUVFRyMjIwIEDB/DBBx9YIz0iIpIxc69ckNtVD1YpFMaOHYvz589jzpw50Gg0CAwMRHZ2tn7BYmlpqcFjRENDQ5Geno7Zs2dj1qxZ8PLyQlZWFvz8/KyRHhERyZjuf82c/nKiEKL1l0ZVVVVwdnaGZ+rrfCgUEVErpLt6FaXJs1FZWQknJyervEf9d8VPP7rC0YyHQlVX6+DTt9yqubYkfNYDERHJSv3VC+b0lxMWCkREJCtaATOfHmm5XFoDFgpERCQrXKMgTau84RIRERE1D44oEBGRrOiggBYKs/rLCQsFIiKSFZ240czpLyeceiAiIiKTOKJARESyojVz6sGcvq0RCwUiIpIVFgrScOqBiIiITOKIAhERyYpOKKATZlz1YEbf1oiFAhERyQqnHqTh1AMRERGZxBEFIiKSFS1soDXj72StBXNpDVgoEBGRrAgz1ygIrlEgIiK6e3GNgjRco0BEREQmcUSBiIhkRStsoBVmrFGQ2bMeWCgQEZGs6KCAzowBdR3kVSlw6oGIiIhM4ogCERHJChczSsNCgYiIZMX8NQqceiAiIiICwBEFIiKSmRuLGc14KBSnHoiIiO5eOjNv4cyrHoiIiIj+hyMKREQkK1zMKA0LBSIikhUdbHjDJQlYKBARkaxohQJaM54AaU7f1ohrFIiIiMgkjigQEZGsaM286kErs6kHjigQEZGs6ISN2e12LF++HD169IBKpUJwcDD279/faHxmZiZ8fHygUqng7++PrVu3GuyfOHEiFAqFQYuIiDCI+eWXXzBu3Dg4OTmhQ4cOmDRpEi5fviwpbxYKREREVrZhwwYkJiYiJSUFhYWFCAgIQHh4OCoqKozG5+XlITY2FpMmTUJRURFiYmIQExODw4cPG8RFRETg3Llz+rZ+/XqD/ePGjcORI0ewbds2bNmyBbt378aUKVMk5c5CgYiIZKV+6sGcJtXChQsxefJkxMfHw9fXF2lpaWjbti1WrlxpNH7JkiWIiIhAUlIS+vbti3nz5mHAgAFYtmyZQZxSqYRarda3jh076vf9+OOPyM7Oxocffojg4GAMHjwYS5cuRUZGBs6ePdvk3FkoEBGRrOjw+5UPt9N0/ztOVVWVQautrTX6fnV1dSgoKEBYWJh+m42NDcLCwpCfn2+0T35+vkE8AISHhzeIz83NRdeuXeHt7Y1p06bh4sWLBsfo0KEDBg4cqN8WFhYGGxsbfPPNN00+XxYvFFJTU3H//ffD0dERXbt2RUxMDI4ePdpon9WrVzeYZ1GpVJZOjYiIyGI8PDzg7Oysb6mpqUbjLly4AK1WC1dXV4Ptrq6u0Gg0RvtoNJpbxkdERODjjz9GTk4O5s+fj127diEyMhJarVZ/jK5duxocw87ODp06dTL5vsZY/KqHXbt2Yfr06bj//vtx/fp1zJo1Cw8//DB++OEHtGvXzmQ/Jycng4JCoZDXdapERNQ8zL/h0o2+ZWVlcHJy0m9XKpVm5ybFE088of93f39/9O/fH71790Zubi5GjBhhsfexeKGQnZ1t8Hr16tXo2rUrCgoK8H//938m+ykUCqjVakunQ0REZMD8Wzjf6Ovk5GRQKJji4uICW1tblJeXG2wvLy83+b2nVqslxQNAr1694OLiguLiYowYMQJqtbrBYsnr16/jl19+kfR9a/U1CpWVlQCATp06NRp3+fJldO/eHR4eHoiOjsaRI0dMxtbW1jaYGyIiImqJHBwcEBQUhJycHP02nU6HnJwchISEGO0TEhJiEA8A27ZtMxkPAKdPn8bFixfh5uamP8alS5dQUFCgj9mxYwd0Oh2Cg4ObnL9VCwWdTofnnnsODzzwAPz8/EzGeXt7Y+XKldi8eTPWrl0LnU6H0NBQnD592mh8amqqwbyQh4eHtT4CERHdZXRQmN2kSkxMxL/+9S+sWbMGP/74I6ZNm4aamhrEx8cDACZMmIDk5GR9/IwZM5CdnY133nkHP/30E+bOnYsDBw4gISEBwI0/rpOSkrBv3z6cPHkSOTk5iI6Oxr333ovw8HAAQN++fREREYHJkydj//79+Prrr5GQkIAnnngC7u7uTc7dqndmnD59Og4fPoy9e/c2GhcSEmJQJYWGhqJv3754//33MW/evAbxycnJSExM1L+uqqpisUBERE1iqakHKcaOHYvz589jzpw50Gg0CAwMRHZ2tn7BYmlpKWxsfj9uaGgo0tPTMXv2bMyaNQteXl7IysrS/9Fta2uL7777DmvWrMGlS5fg7u6Ohx9+GPPmzTNYK7Fu3TokJCRgxIgRsLGxweOPP453331XUu4KIazzvMyEhARs3rwZu3fvRs+ePSX3Hz16NOzs7BrcPMKYqqoqODs7wzP1ddjwagkiolZHd/UqSpNno7Kysknz/rej/rvi7QOD0ab97f+d/Nvl63hh4F6r5tqSWHzqQQiBhIQEbNq0CTt27LitIkGr1eL777/Xz7MQERHRnWHxqYfp06cjPT0dmzdvhqOjo/5aTWdnZ7Rp0wbAjbmYbt266a85fe211/CnP/0J9957Ly5duoQFCxbg1KlTeOqppyydHhERyZxOKKAz41HR5vRtjSxeKKxYsQIAMHToUIPtq1atwsSJEwE0nIv59ddfMXnyZGg0GnTs2BFBQUHIy8uDr6+vpdMjIiKZ05n59Ehz7sHQGlm8UGjKkofc3FyD14sWLcKiRYssnQoRERGZyapXPRAREbU05jwqur6/nLBQICIiWdFCAe1t3Avh5v5yIq+yiIiIiCThiAIREckKpx6kYaFARESyooV50wday6XSKsirLCIiIiJJOKJARESywqkHaVgoEBGRrNyJh0K1ZiwUiIhIVsRtPir65v5yIq+yiIiIiCThiAIREckKpx6kYaFARESywqdHSiOvsoiIiIgk4YgCERHJitbMx0yb07c1YqFARESywqkHaeRVFhEREZEkHFEgIiJZ0cEGOjP+Tjanb2vEQoGIiGRFKxTQmjF9YE7f1kheZRERERFJwhEFIiKSFS5mlIaFAhERyYow8+mRgndmJCIiuntpoYDWjAc7mdO3NZJXWURERESScESBiIhkRSfMW2egExZMphVgoUBERLKiM3ONgjl9WyN5fVoiIiKShCMKREQkKzoooDNjQaI5fVsjFgpERCQrvDOjNJx6ICIiIpM4okBERLLCxYzSsFAgIiJZ0cHMWzjLbI2CvMoiIiIikoQjCkREJCvCzKsehMxGFFgoEBGRrPDpkdKwUCAiIlnhYkZpLP5p586dC4VCYdB8fHwa7ZOZmQkfHx+oVCr4+/tj69atlk6LiIiIboNVyqJ+/frh3Llz+rZ3716TsXl5eYiNjcWkSZNQVFSEmJgYxMTE4PDhw9ZIjYiIZK5+6sGcJidWKRTs7OygVqv1zcXFxWTskiVLEBERgaSkJPTt2xfz5s3DgAEDsGzZMmukRkREMld/C2dzmpxYpVA4fvw43N3d0atXL4wbNw6lpaUmY/Pz8xEWFmawLTw8HPn5+Sb71NbWoqqqyqARERGR5Vm8UAgODsbq1auRnZ2NFStWoKSkBA8++CCqq6uNxms0Gri6uhpsc3V1hUajMfkeqampcHZ21jcPDw+LfgYiIrp7cepBGosXCpGRkRg9ejT69++P8PBwbN26FZcuXcKnn35qsfdITk5GZWWlvpWVlVns2EREdHdjoSCN1S+P7NChA/r06YPi4mKj+9VqNcrLyw22lZeXQ61WmzymUqmEUqm0aJ5ERETUkNUvBr18+TJOnDgBNzc3o/tDQkKQk5NjsG3btm0ICQmxdmpERCRDHFGQxuKFwgsvvIBdu3bh5MmTyMvLw6hRo2Bra4vY2FgAwIQJE5CcnKyPnzFjBrKzs/HOO+/gp59+wty5c3HgwAEkJCRYOjUiIiIWChJZfOrh9OnTiI2NxcWLF9GlSxcMHjwY+/btQ5cuXQAApaWlsLH5vT4JDQ1Feno6Zs+ejVmzZsHLywtZWVnw8/OzdGpEREQkkcULhYyMjEb35+bmNtg2evRojB492tKpEBERNSBg3qOiheVSaRX4rAciIpIVPhRKGhYKREQkKywUpJHXI7CIiIhIEo4oEBGRrHBEQRoWCkREJCssFKTh1AMRERGZxBEFIiKSFSEUEGaMCpjTtzVioUBERLKig8Ks+yiY07c14tQDERERmcQRBSIikhUuZpSGhQIREckK1yhIw6kHIiIiMokjCkREJCucepCGhQIREckKpx6kYaFARESyIswcUZBbocA1CkRERGQSRxSIiEhWBAAhzOsvJywUiIhIVnRQQME7MzYZpx6IiIiawfLly9GjRw+oVCoEBwdj//79jcZnZmbCx8cHKpUK/v7+2Lp1q8nYqVOnQqFQYPHixQbbjx07hujoaLi4uMDJyQmDBw/Gzp07JeXNQoGIiGSl/qoHc5pUGzZsQGJiIlJSUlBYWIiAgACEh4ejoqLCaHxeXh5iY2MxadIkFBUVISYmBjExMTh8+HCD2E2bNmHfvn1wd3dvsO/Pf/4zrl+/jh07dqCgoAABAQH485//DI1G0+TcWSgQEZGs1N9HwZwm1cKFCzF58mTEx8fD19cXaWlpaNu2LVauXGk0fsmSJYiIiEBSUhL69u2LefPmYcCAAVi2bJlB3JkzZ/DMM89g3bp1sLe3N9h34cIFHD9+HC+99BL69+8PLy8vvPnmm7hy5YrRgsMUFgpERES3oaqqyqDV1tYajaurq0NBQQHCwsL022xsbBAWFob8/HyjffLz8w3iASA8PNwgXqfTYfz48UhKSkK/fv0aHKNz587w9vbGxx9/jJqaGly/fh3vv/8+unbtiqCgoCZ/ThYKREQkK0KY3wDAw8MDzs7O+paammr0/S5cuACtVgtXV1eD7a6urianADQazS3j58+fDzs7Ozz77LNGj6FQKLB9+3YUFRXB0dERKpUKCxcuRHZ2Njp27NjU08WrHoiISF4sdWfGsrIyODk56bcrlUqzc2uqgoICLFmyBIWFhVAojH8WIQSmT5+Orl27Ys+ePWjTpg0+/PBDjBw5Et9++y3c3Nya9F4cUSAiIroNTk5OBs1UoeDi4gJbW1uUl5cbbC8vL4darTbaR61WNxq/Z88eVFRUwNPTE3Z2drCzs8OpU6fw/PPPo0ePHgCAHTt2YMuWLcjIyMADDzyAAQMG4L333kObNm2wZs2aJn9OFgpERCQrzX3Vg4ODA4KCgpCTk6PfptPpkJOTg5CQEKN9QkJCDOIBYNu2bfr48ePH47vvvsPBgwf1zd3dHUlJSfjyyy8BAFeuXAFwYz3EzWxsbKDT6ZqcP6ceiIhIVnRCAUUzPz0yMTERcXFxGDhwIAYNGoTFixejpqYG8fHxAIAJEyagW7du+nUOM2bMwJAhQ/DOO+8gKioKGRkZOHDgAD744AMANxYqdu7c2eA97O3toVar4e3tDeBGsdGxY0fExcVhzpw5aNOmDf71r3+hpKQEUVFRTc6dhQIREcnKzQsSb7e/VGPHjsX58+cxZ84caDQaBAYGIjs7W79gsbS01OAv/9DQUKSnp2P27NmYNWsWvLy8kJWVBT8/vya/p4uLC7Kzs/Hyyy9j+PDhuHbtGvr164fNmzcjICCgycdRCGHO6WoZqqqq4OzsDM/U12GjUt3pdIiISCLd1asoTZ6NyspKgwWCllT/XdFn3UuwbXv7Cw+1V2pxbNybVs21JeGIAhERycqNEQVzrnqwYDKtAAsFIiKSFUtdHikXvOqBiIiITOKIAhERyYr4XzOnv5ywUCAiIlnh1IM0nHogIiIikziiQERE8sK5B0ksPqLQo0cPKBSKBm369OlG41evXt0gVsV7IRARkbWYe/tmmU09WHxE4dtvv4VWq9W/Pnz4MB566CGMHj3aZB8nJyccPXpU/9rUk7CIiIjMdSfuzNiaWbxQ6NKli8HrN998E71798aQIUNM9lEoFCafoEVERER3jlUXM9bV1WHt2rV48sknGx0luHz5Mrp37w4PDw9ER0fjyJEjjR63trYWVVVVBo2IiKgpmvvpka2dVQuFrKwsXLp0CRMnTjQZ4+3tjZUrV2Lz5s1Yu3YtdDodQkNDcfr0aZN9UlNT4ezsrG8eHh5WyJ6IiO5K9esMzGkyYtVC4aOPPkJkZCTc3d1NxoSEhGDChAkIDAzEkCFDsHHjRnTp0gXvv/++yT7JycmorKzUt7KyMmukT0REJHtWuzzy1KlT2L59OzZu3Cipn729Pe677z4UFxebjFEqlVAqb//JX0REJF9czCiN1UYUVq1aha5duyIqKkpSP61Wi++//x5ubm5WyoyIiGRNWKDJiFUKBZ1Oh1WrViEuLg52doaDFhMmTEBycrL+9WuvvYavvvoKP//8MwoLC/G3v/0Np06dwlNPPWWN1IiIiEgCq0w9bN++HaWlpXjyyScb7CstLYWNze/1ya+//orJkydDo9GgY8eOCAoKQl5eHnx9fa2RGhERyRyf9SCNVQqFhx9+GMLEJE5ubq7B60WLFmHRokXWSIOIiMg4mU0fmIMPhSIiIiKT+FAoIiKSFU49SMNCgYiI5IVPj5SEhQIREcmM4n/NnP7ywTUKREREZBJHFIiISF449SAJCwUiIpIXFgqScOqBiIiITOKIAhERyYu5j4rm5ZFERER3Lz49UhpOPRAREZFJHFEgIiJ54WJGSVgoEBGRvHCNgiSceiAiIiKTOKJARESyohA3mjn95YSFAhERyQvXKEjCQoGIiOSFaxQk4RoFIiIiMokjCkREJC+cepCEhQIREckLCwVJOPVAREREJnFEgYiI5IUjCpKwUCAiInnhVQ+ScOqBiIiITOKIAhERyQrvzCgNCwUiIpIXrlGQhFMPREREZBILBSIiIjKJUw9ERCQrCpi5RsFimbQOLBSIiEheeHmkJJx6ICIiIpM4okBERPLCqx4kYaFARETywkJBEk49EBERkUkcUSAiIlnhnRmlkTyisHv3bowcORLu7u5QKBTIysoy2C+EwJw5c+Dm5oY2bdogLCwMx48fv+Vxly9fjh49ekClUiE4OBj79++XmhoREdGtCQs0GZFcKNTU1CAgIADLly83uv+tt97Cu+++i7S0NHzzzTdo164dwsPDcfXqVZPH3LBhAxITE5GSkoLCwkIEBAQgPDwcFRUVUtMjIiIiC5JcKERGRuL111/HqFGjGuwTQmDx4sWYPXs2oqOj0b9/f3z88cc4e/Zsg5GHmy1cuBCTJ09GfHw8fH19kZaWhrZt22LlypVS0yMiImocRxQksehixpKSEmg0GoSFhem3OTs7Izg4GPn5+Ub71NXVoaCgwKCPjY0NwsLCTPapra1FVVWVQSMiImqK+jUK5jQ5sWihoNFoAACurq4G211dXfX7/ujChQvQarWS+qSmpsLZ2VnfPDw8LJA9ERER/VGrvDwyOTkZlZWV+lZWVnanUyIiotai/hbO5jQZsejlkWq1GgBQXl4ONzc3/fby8nIEBgYa7ePi4gJbW1uUl5cbbC8vL9cf74+USiWUSqVlkiYiInnhDZckseiIQs+ePaFWq5GTk6PfVlVVhW+++QYhISFG+zg4OCAoKMigj06nQ05Ojsk+REREt4trFKSRPKJw+fJlFBcX61+XlJTg4MGD6NSpEzw9PfHcc8/h9ddfh5eXF3r27IlXXnkF7u7uiImJ0fcZMWIERo0ahYSEBABAYmIi4uLiMHDgQAwaNAiLFy9GTU0N4uPjzf+EREREdNskFwoHDhzAsGHD9K8TExMBAHFxcVi9ejVefPFF1NTUYMqUKbh06RIGDx6M7OxsqFQqfZ8TJ07gwoUL+tdjx47F+fPnMWfOHGg0GgQGBiI7O7vBAkciIiKzcepBEoUQotV/5KqqKjg7O8Mz9XXY3FSQEBFR66C7ehWlybNRWVkJJycnq7xH/XdFr1f+CVszviu0V6/i53mzrJprS9Iqr3ogIiKi5sGHQhERkbxw6kESFgpERCQvLBQk4dQDERERmcRCgYiIZOVO3Udh+fLl6NGjB1QqFYKDg7F///5G4zMzM+Hj4wOVSgV/f39s3brVZOzUqVOhUCiwePHiBvu++OILBAcHo02bNujYsaPB7QqagoUCERGRlW3YsAGJiYlISUlBYWEhAgICEB4ejoqKCqPxeXl5iI2NxaRJk1BUVISYmBjExMTg8OHDDWI3bdqEffv2wd3dvcG+zz77DOPHj0d8fDwOHTqEr7/+Gn/9618l5c5CgYiIyMoWLlyIyZMnIz4+Hr6+vkhLS0Pbtm2xcuVKo/FLlixBREQEkpKS0LdvX8ybNw8DBgzAsmXLDOLOnDmDZ555BuvWrYO9vb3BvuvXr2PGjBlYsGABpk6dij59+sDX1xdjxoyRlDsLBSIikhdhgYYb92W4udXW1hp9u7q6OhQUFCAsLEy/zcbGBmFhYcjPzzfaJz8/3yAeAMLDww3idTodxo8fj6SkJPTr16/BMQoLC3HmzBnY2Njgvvvug5ubGyIjI42OSjSGhQIREcmKpdYoeHh4wNnZWd9SU1ONvt+FCxeg1Wob3G3Y1dUVGo3GaB+NRnPL+Pnz58POzg7PPvus0WP8/PPPAIC5c+di9uzZ2LJlCzp27IihQ4fil19+adK5Anh5JBERyZEFLnEsKyszuDNjcz7VuKCgAEuWLEFhYSEUCuOPvdbpdACAl19+GY8//jgAYNWqVbjnnnuQmZmJp59+uknvxREFIiKi2+Dk5GTQTBUKLi4usLW1RXl5ucH28vJyqNVqo33UanWj8Xv27EFFRQU8PT1hZ2cHOzs7nDp1Cs8//zx69OgBAHBzcwMA+Pr66o+hVCrRq1cvlJaWNvlzslAgIiJ5sdAahaZycHBAUFAQcnJy9Nt0Oh1ycnIQEhJitE9ISIhBPABs27ZNHz9+/Hh89913OHjwoL65u7sjKSkJX375JQAgKCgISqUSR48e1R/j2rVrOHnyJLp3797k/Dn1QEREsmLOvRDq+0uVmJiIuLg4DBw4EIMGDcLixYtRU1OD+Ph4AMCECRPQrVs3/TqHGTNmYMiQIXjnnXcQFRWFjIwMHDhwAB988AEAoHPnzujcubPBe9jb20OtVsPb2xvAjRGPqVOnIiUlBR4eHujevTsWLFgAABg9enSTc2ehQEREZGVjx47F+fPnMWfOHGg0GgQGBiI7O1u/YLG0tBQ2Nr8P8oeGhiI9PR2zZ8/GrFmz4OXlhaysLPj5+Ul63wULFsDOzg7jx4/Hb7/9huDgYOzYsQMdO3Zs8jH4mGkiIrrjmvMx015J/4St0ozHTNdexfEF8nnMNEcUiIhIVu7E1ENrxsWMREREZBJHFIiISF74mGlJWCgQEZG8sFCQhFMPREREZBJHFIiISFa4mFEaFgpERCQvnHqQhIUCERHJCwsFSbhGgYiIiEziiAIREckK1yhIw0KBiIjkhVMPknDqgYiIiEziiAIREckKpx6kYaFARETywqkHSTj1QERERCZxRIGIiOSFIwqSsFAgIiJZUfyvmdNfTjj1QERERCZxRIGIiOSFUw+SsFAgIiJZ4eWR0kieeti9ezdGjhwJd3d3KBQKZGVl6fddu3YNM2fOhL+/P9q1awd3d3dMmDABZ8+ebfSYc+fOhUKhMGg+Pj6SPwwREdEtCQs0GZFcKNTU1CAgIADLly9vsO/KlSsoLCzEK6+8gsLCQmzcuBFHjx7Fo48+esvj9uvXD+fOndO3vXv3Sk2NiIiILEzy1ENkZCQiIyON7nN2dsa2bdsMti1btgyDBg1CaWkpPD09TSdiZwe1Wi01HSIiIulkNipgDqtf9VBZWQmFQoEOHTo0Gnf8+HG4u7ujV69eGDduHEpLS03G1tbWoqqqyqARERE1Rf0aBXOanFi1ULh69SpmzpyJ2NhYODk5mYwLDg7G6tWrkZ2djRUrVqCkpAQPPvggqqurjcanpqbC2dlZ3zw8PKz1EYiIiGTNaoXCtWvXMGbMGAghsGLFikZjIyMjMXr0aPTv3x/h4eHYunUrLl26hE8//dRofHJyMiorK/WtrKzMGh+BiIjuRlzMKIlVLo+sLxJOnTqFHTt2NDqaYEyHDh3Qp08fFBcXG92vVCqhVCotkSoREckML4+UxuIjCvVFwvHjx7F9+3Z07txZ8jEuX76MEydOwM3NzdLpERERkQSSC4XLly/j4MGDOHjwIACgpKQEBw8eRGlpKa5du4a//OUvOHDgANatWwetVguNRgONRoO6ujr9MUaMGIFly5bpX7/wwgvYtWsXTp48iby8PIwaNQq2traIjY01/xMSERHdjFMPkkieejhw4ACGDRumf52YmAgAiIuLw9y5c/H5558DAAIDAw367dy5E0OHDgUAnDhxAhcuXNDvO336NGJjY3Hx4kV06dIFgwcPxr59+9ClSxep6RERETWKUw/SSC4Uhg4dCiFMn6XG9tU7efKkweuMjAypaRAREVEz4LMeiIhIXvhQKElYKBARkbywUJCEhQIREckK1yhIY/VbOBMREVHrxREFIiKSF049SMJCgYiIZEUhBBRNuEKvsf5ywqkHIiIiMokjCkREJC+cepCEhQIREckKr3qQhlMPREREZBJHFIiISF449SAJCwUiIpIVTj1Iw6kHIiIiMokjCkREJC+cepCEhQIREckKpx6kYaFARETywhEFSbhGgYiIiEziiAIREcmO3KYPzMFCgYiI5EWIG82c/jLCqQciIiIyiSMKREQkK7zqQRoWCkREJC+86kESTj0QERGRSRxRICIiWVHobjRz+ssJCwUiIpIXTj1IwqkHIiIiMokjCkREJCu86kEaFgpERCQvvOGSJCwUiIhIVjiiIA3XKBAREZFJHFEgIiJ54VUPkrBQICIiWeHUgzSceiAiIiKTOKJARETywqseJGGhQEREssKpB2kkTz3s3r0bI0eOhLu7OxQKBbKysgz2T5w4EQqFwqBFRETc8rjLly9Hjx49oFKpEBwcjP3790tNjYiIiCxMcqFQU1ODgIAALF++3GRMREQEzp07p2/r169v9JgbNmxAYmIiUlJSUFhYiICAAISHh6OiokJqekRERI0TFmgyInnqITIyEpGRkY3GKJVKqNXqJh9z4cKFmDx5MuLj4wEAaWlp+OKLL7By5Uq89NJLUlMkIiIyiVMP0ljlqofc3Fx07doV3t7emDZtGi5evGgytq6uDgUFBQgLC/s9KRsbhIWFIT8/32if2tpaVFVVGTQiIqKWTOoUe2ZmJnx8fKBSqeDv74+tW7eajJ06dSoUCgUWL15sdH9tbS0CAwOhUChw8OBBSXlbvFCIiIjAxx9/jJycHMyfPx+7du1CZGQktFqt0fgLFy5Aq9XC1dXVYLurqys0Go3RPqmpqXB2dtY3Dw8PS38MIiK6W+mE+U0iqVPseXl5iI2NxaRJk1BUVISYmBjExMTg8OHDDWI3bdqEffv2wd3d3eT7v/jii43ub4zFC4UnnngCjz76KPz9/RETE4MtW7bg22+/RW5ursXeIzk5GZWVlfpWVlZmsWMTEdFdzkJrFP44sl1bW2vyLW+eYvf19UVaWhratm2LlStXGo1fsmQJIiIikJSUhL59+2LevHkYMGAAli1bZhB35swZPPPMM1i3bh3s7e2NHuu///0vvvrqK7z99ttNOz9/YPUbLvXq1QsuLi4oLi42ut/FxQW2trYoLy832F5eXm5ynYNSqYSTk5NBIyIiagoFfl+ncFvtf8fx8PAwGN1OTU01+n63M8Wen59vEA8A4eHhBvE6nQ7jx49HUlIS+vXrZ/Q45eXlmDx5Mj755BO0bdu26SfpJlYvFE6fPo2LFy/Czc3N6H4HBwcEBQUhJydHv02n0yEnJwchISHWTo+IiOi2lJWVGYxuJycnG427nSl2jUZzy/j58+fDzs4Ozz77rNFjCCEwceJETJ06FQMHDpTy0QxIvurh8uXLBqMDJSUlOHjwIDp16oROnTrh1VdfxeOPPw61Wo0TJ07gxRdfxL333ovw8HB9nxEjRmDUqFFISEgAACQmJiIuLg4DBw7EoEGDsHjxYtTU1OivgiAiIrIYC92Z8U6OaBcUFGDJkiUoLCyEQqEwGrN06VJUV1ebLGCaSnKhcODAAQwbNkz/OjExEQAQFxeHFStW4LvvvsOaNWtw6dIluLu74+GHH8a8efOgVCr1fU6cOIELFy7oX48dOxbnz5/HnDlzoNFoEBgYiOzs7AbVFBERkbma+/LI25liV6vVjcbv2bMHFRUV8PT01O/XarV4/vnnsXjxYpw8eRI7duxAfn6+wfcvAAwcOBDjxo3DmjVrmpS/QojWf9PqqqoqODs7wzP1ddioVHc6HSIikkh39SpKk2ejsrLSan+l139XDB4+F3Z2t/9dcf36VezdMVdSrsHBwRg0aBCWLl0K4MYUu6enJxISEozeL2js2LG4cuUK/vOf/+i3hYaGon///khLS8PFixdx7tw5gz7h4eEYP3484uPj4e3tjdLSUoPbB5w9exbh4eH497//jeDgYNxzzz1Nyp3PeiAiInkx9+6Kt9H3VlPsEyZMQLdu3fQLImfMmIEhQ4bgnXfeQVRUFDIyMnDgwAF88MEHAIDOnTujc+fOBu9hb28PtVoNb29vADAYbQCA9u3bAwB69+7d5CIBYKFAREQyoxACCjMG02+n762m2EtLS2Fj8/v1BaGhoUhPT8fs2bMxa9YseHl5ISsrC35+fred9+3i1AMREd1xzTn18ODQFLOnHvbkvmrVXFsSjigQEZG86P7XzOkvIywUiIhIVu7E1ENrZvUbLhEREVHrxREFIiKSlztw1UNrxkKBiIjkxUJ3ZpQLFgpERCQrzX1nxtaOaxSIiIjIJI4oEBGRvHDqQRIWCkREJCsK3Y1mTn854dQDERERmcQRBSIikhdOPUjCQoGIiOSF91GQhFMPREREZBJHFIiISFb4rAdpWCgQEZG8cI2CJJx6ICIiIpM4okBERPIiAJhzLwR5DSiwUCAiInnhGgVpWCgQEZG8CJi5RsFimbQKXKNAREREJnFEgYiI5IVXPUjCQoGIiORFB0BhZn8Z4dQDERERmcQRBSIikhVe9SANCwUiIpIXrlGQhFMPREREZBJHFIiISF44oiAJCwUiIpIXFgqScOqBiIiITOKIAhERyQvvoyAJCwUiIpIVXh4pDQsFIiKSF65RkETyGoXdu3dj5MiRcHd3h0KhQFZWlsF+hUJhtC1YsMDkMefOndsg3sfHR/KHISIiIsuSPKJQU1ODgIAAPPnkk3jsscca7D937pzB6//+97+YNGkSHn/88UaP269fP2zfvv33xOw42EFERFagE4DCjFEBnbxGFCR/G0dGRiIyMtLkfrVabfB68+bNGDZsGHr16tV4InZ2DfoSERFZHKceJLHq5ZHl5eX44osvMGnSpFvGHj9+HO7u7ujVqxfGjRuH0tJSk7G1tbWoqqoyaERERGR5Vi0U1qxZA0dHR6NTFDcLDg7G6tWrkZ2djRUrVqCkpAQPPvggqqurjcanpqbC2dlZ3zw8PKyRPhER3ZXE76MKt9PAEQWLWblyJcaNGweVStVoXGRkJEaPHo3+/fsjPDwcW7duxaVLl/Dpp58ajU9OTkZlZaW+lZWVWSN9IiK6G5lTJJg7bdEKWW3F4J49e3D06FFs2LBBct8OHTqgT58+KC4uNrpfqVRCqVSamyIRERHdgtVGFD766CMEBQUhICBAct/Lly/jxIkTcHNzs0JmREQkazphfpMRyYXC5cuXcfDgQRw8eBAAUFJSgoMHDxosPqyqqkJmZiaeeuopo8cYMWIEli1bpn/9wgsvYNeuXTh58iTy8vIwatQo2NraIjY2Vmp6REREjRM685uMSJ56OHDgAIYNG6Z/nZiYCACIi4vD6tWrAQAZGRkQQpj8oj9x4gQuXLigf3369GnExsbi4sWL6NKlCwYPHox9+/ahS5cuUtMjIiIiC5JcKAwdOhTiFgs5pkyZgilTppjcf/LkSYPXGRkZUtMgIiK6PbyPgiS8/SEREcmLzsxLHGW2RoGFAhERyQtHFCSx6n0UiIiIqHXjiAIREcmLgJkjChbLpFVgoUBERPLCqQdJOPVAREREJnFEgYiI5EWnA2DGTZN0vOESERHR3YtTD5Jw6oGIiIhM4ogCERHJC0cUJGGhQERE8sI7M0rCqQciIiIyiSMKREQkK0LoIMx4VLQ5fVsjFgpERCQvQpg3fcA1CkRERHcxYeYaBZkVClyjQERERCZxRIGIiORFpwMUZqwz4BoFIiKiuxinHiTh1AMRERGZxBEFIiKSFaHTQZgx9cDLI4mIiO5mnHqQhFMPREREZBJHFIiISF50AlBwRKGpWCgQEZG8CAHAnMsj5VUocOqBiIiITGKhQEREsiJ0wux2O5YvX44ePXpApVIhODgY+/fvbzQ+MzMTPj4+UKlU8Pf3x9atW03GTp06FQqFAosXL9ZvO3nyJCZNmoSePXuiTZs26N27N1JSUlBXVycpbxYKREQkL0JnfpNow4YNSExMREpKCgoLCxEQEIDw8HBUVFQYjc/Ly0NsbCwmTZqEoqIixMTEICYmBocPH24Qu2nTJuzbtw/u7u4G23/66SfodDq8//77OHLkCBYtWoS0tDTMmjVLUu4KIVr/ZEtVVRWcnZ3hmfo6bFSqO50OERFJpLt6FaXJs1FZWQknJyervEf9d8VQxSjYKexv+zjXxTXkik2Scg0ODsb999+PZcuWAQB0Oh08PDzwzDPP4KWXXmoQP3bsWNTU1GDLli36bX/6058QGBiItLQ0/bYzZ84gODgYX375JaKiovDcc8/hueeeM5nHggULsGLFCvz8889N/LQcUSAiIrotVVVVBq22ttZoXF1dHQoKChAWFqbfZmNjg7CwMOTn5xvtk5+fbxAPAOHh4QbxOp0O48ePR1JSEvr169eknCsrK9GpU6cmxda7K656qB8U0V29eoczISKi21H//+/mGOS+LmrNerDTdVwDAHh4eBhsT0lJwdy5cxvEX7hwAVqtFq6urgbbXV1d8dNPPxl9D41GYzReo9HoX8+fPx92dnZ49tlnm5R3cXExli5dirfffrtJ8fXuikKhuroaAHD61dfvcCZERGSO6upqODs7W+XYDg4OUKvV2KsxvSiwqdRqNQ4dOgTVTdPdSqXS7OM2VUFBAZYsWYLCwkIoFIpbxp85cwYREREYPXo0Jk+eLOm97opCwd3dHWVlZXB0dGz0hFVVVcHDwwNlZWVWmwOzBubdvFpr3kDrzZ15N6+WmLcQAtXV1Q0W5FmSSqVCSUmJ5FX/xjg4OBgUCY1xcXGBra0tysvLDbaXl5dDrVYb7aNWqxuN37NnDyoqKuDp6anfr9Vq8fzzz2Px4sU4efKkfvvZs2cxbNgwhIaG4oMPPmhSzje7KwoFGxsb3HPPPU2Od3JyajG/HFIw7+bVWvMGWm/uzLt5tbS8rTWScDOVStXkL3hLcXBwQFBQEHJychATEwPgxvqCnJwcJCQkGO0TEhKCnJwcg4WJ27ZtQ0hICABg/PjxRtcwjB8/HvHx8fptZ86cwbBhwxAUFIRVq1bBxkb60sS7olAgIiJqyRITExEXF4eBAwdi0KBBWLx4MWpqavRf6hMmTEC3bt2QmpoKAJgxYwaGDBmCd955B1FRUcjIyMCBAwf0IwKdO3dG586dDd7D3t4earUa3t7eAG4UCUOHDkX37t3x9ttv4/z58/pYUyMZxrBQICIisrKxY8fi/PnzmDNnDjQaDQIDA5Gdna1fsFhaWmrw135oaCjS09Mxe/ZszJo1C15eXsjKyoKfn1+T33Pbtm0oLi5GcXFxg1F3KYtGZVUoKJVKpKSkNOuCE0tg3s2rteYNtN7cmXfzaq15t3YJCQkmpxpyc3MbbBs9ejRGjx7d5OPfvC4BACZOnIiJEydKyNC4u+KGS0RERGQdvOESERERmcRCgYiIiExioUBEREQmsVAgIiIik1goEBERkUl3XaGwfPly9OjRAyqVCsHBwdi/f3+j8ZmZmfDx8YFKpYK/vz+2bjX/HuBSpKam4v7774ejoyO6du2KmJgYHD16tNE+q1evhkKhMGjNfaexuXPnNsjBx8en0T53+lwDQI8ePRrkrVAoMH36dKPxd/Jc7969GyNHjoS7uzsUCgWysrIM9gshMGfOHLi5uaFNmzYICwvD8ePHb3lcqb8jlsz72rVrmDlzJvz9/dGuXTu4u7tjwoQJOHv2bKPHvJ2fN0vmDdy41OyPOURERNzyuHfyfAMw+vOuUCiwYMECk8dsjvNNrcddVShs2LABiYmJSElJQWFhIQICAhAeHo6Kigqj8Xl5eYiNjcWkSZNQVFSEmJgYxMTE4PDhw82W865duzB9+nTs27cP27Ztw7Vr1/Dwww+jpqam0X5OTk44d+6cvp06daqZMv5dv379DHLYu3evydiWcK4B4NtvvzXIedu2bQDQ6LXKd+pc19TUICAgAMuXLze6/6233sK7776LtLQ0fPPNN2jXrh3Cw8NxtZGnqEr9HbF03leuXEFhYSFeeeUVFBYWYuPGjTh69CgeffTRWx5Xys+bpfOuFxERYZDD+vXrGz3mnT7fAAzyPXfuHFauXAmFQoHHH3+80eNa+3xTKyLuIoMGDRLTp0/Xv9ZqtcLd3V2kpqYajR8zZoyIiooy2BYcHCyefvppq+bZmIqKCgFA7Nq1y2TMqlWrhLOzc/MlZURKSooICAhocnxLPNdCCDFjxgzRu3dvodPpjO5vCedaCCEAiE2bNulf63Q6oVarxYIFC/TbLl26JJRKpVi/fr3J40j9HbF03sbs379fABCnTp0yGSP1581cxvKOi4sT0dHRko7TEs93dHS0GD58eKMxzX2+qWW7a0YU6urqUFBQYPCQDBsbG4SFhSE/P99on/z8fKMP1TAV3xwqKysBAJ06dWo07vLly+jevTs8PDwQHR2NI0eONEd6Bo4fPw53d3f06tUL48aNQ2lpqcnYlniu6+rqsHbtWjz55JONPnW0JZzrPyopKYFGozE4p87OzggODjZ5Tm/nd6Q5VFZWQqFQoEOHDo3GSfl5s5bc3Fx07doV3t7emDZtGi5evGgytiWe7/LycnzxxReYNGnSLWNbwvmmluGuKRQuXLgArVarv292PVdXV2g0GqN9NBqNpHhr0+l0eO655/DAAw80ej9vb29vrFy5Eps3b8batWuh0+kQGhqK06dPN1uuwcHBWL16NbKzs7FixQqUlJTgwQcfRHV1tdH4lnauASArKwuXLl1q9BanLeFcG1N/3qSc09v5HbG2q1evYubMmYiNjW30KYZSf96sISIiAh9//DFycnIwf/587Nq1C5GRkdBqtUbjW+L5XrNmDRwdHfHYY481GtcSzje1HLJ61kNLN336dBw+fPiWc4EhISH6R40CNx4e0rdvX7z//vuYN2+etdMEAERGRur/vX///ggODkb37t3x6aefNumvlZbgo48+QmRkJNzd3U3GtIRzfbe6du0axowZAyEEVqxY0WhsS/h5e+KJJ/T/7u/vj/79+6N3797Izc3FiBEjmiUHc61cuRLjxo275YLclnC+qeW4a0YUXFxcYGtri/LycoPt5eXlJh+nqVarJcVbU0JCArZs2YKdO3c2eMrXrdjb2+O+++5DcXGxlbK7tQ4dOqBPnz4mc2hJ5xoATp06he3bt+Opp56S1K8lnGvg90fESjmnt/M7Yi31RcKpU6ewbdu2RkcTjLnVz1tz6NWrF1xcXEzm0JLONwDs2bMHR48elfwzD7SM8013zl1TKDg4OCAoKAg5OTn6bTqdDjk5OQZ/Ed4sJCTEIB648VhOU/HWIIRAQkICNm3ahB07dqBnz56Sj6HVavH999/Dzc3NChk2zeXLl3HixAmTObSEc32zVatWoWvXroiKipLUryWcawDo2bMn1Gq1wTmtqqrCN998Y/Kc3s7viDXUFwnHjx/H9u3b0blzZ8nHuNXPW3M4ffo0Ll68aDKHlnK+63300UcICgpCQECA5L4t4XzTHXSnV1NaUkZGhlAqlWL16tXihx9+EFOmTBEdOnQQGo1GCCHE+PHjxUsvvaSP//rrr4WdnZ14++23xY8//ihSUlKEvb29+P7775st52nTpglnZ2eRm5srzp07p29XrlzRx/wx71dffVV8+eWX4sSJE6KgoEA88cQTQqVSiSNHjjRb3s8//7zIzc0VJSUl4uuvvxZhYWHCxcVFVFRUGM25JZzrelqtVnh6eoqZM2c22NeSznV1dbUoKioSRUVFAoBYuHChKCoq0l8d8Oabb4oOHTqIzZs3i++++05ER0eLnj17it9++01/jOHDh4ulS5fqX9/qd8TaedfV1YlHH31U3HPPPeLgwYMGP/O1tbUm877Vz5u1866urhYvvPCCyM/PFyUlJWL79u1iwIABwsvLS1y9etVk3nf6fNerrKwUbdu2FStWrDB6jDtxvqn1uKsKBSGEWLp0qfD09BQODg5i0KBBYt++ffp9Q4YMEXFxcQbxn376qejTp49wcHAQ/fr1E1988UWz5gvAaFu1apXJvJ977jn9Z3R1dRWPPPKIKCwsbNa8x44dK9zc3ISDg4Po1q2bGDt2rCguLjaZsxB3/lzX+/LLLwUAcfTo0Qb7WtK53rlzp9Gfjfr8dDqdeOWVV4Srq6tQKpVixIgRDT5T9+7dRUpKisG2xn5HrJ13SUmJyZ/5nTt3msz7Vj9v1s77ypUr4uGHHxZdunQR9vb2onv37mLy5MkNvvBb2vmu9/7774s2bdqIS5cuGT3GnTjf1HoohBDCqkMWRERE1GrdNWsUiIiIyPJYKBAREZFJLBSIiIjIJBYKREREZBILBSIiIjKJhQIRERGZxEKBiIiITGKhQERERCaxUCAiIiKTWCgQERGRSSwUiIiIyKT/BzXA9huXFxRaAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGzCAYAAABzfl4TAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVHlJREFUeJzt3XtcVGX+B/DPoDJ4GxAFBgpFzARvoJiImTdIUCtJ8sLSeiNpWzGVLmpbalmLrtdWXcnNW6uEkmalLoUoWoKaIJaGpIaCyuAtQDAuMuf3h8v5Oc4Fhpkj6Pm8f6/n9XPOeZ7nPOcEO1+e21EIgiCAiIiIyApsGroBRERE9OhgYEFERERWw8CCiIiIrIaBBREREVkNAwsiIiKyGgYWREREZDUMLIiIiMhqGFgQERGR1TCwICIiIqthYEGyMGnSJHh4eDR0Mx6IwYMHY/DgwQ12fYVCgejo6Aa7PhE1LAYWJKnc3FxER0fjySefRIsWLdCiRQt07doV06ZNw08//dTQzTOqpKQE77//Pnx8fNCqVSs0b94c3bt3x+zZs3HlyhWDZcaOHQuFQoHZs2cbPJ+amgqFQgGFQoEtW7YYzPP0009DoVCge/fuVrsXKaSlpWHBggUoKip64NdOSUnBlClTxJ8pT09PvPLKKygoKNDL+/e//x39+vWDk5MT7Ozs0LlzZ8ycORPXrl174O0mkoumDd0AenTt3r0b48aNQ9OmTREREQEfHx/Y2NjgzJkz2LlzJ9auXYvc3Fx06NChoZuq47fffkNQUBDy8vIwZswYREVFwdbWFj/99BPWr1+PL7/8Er/++qtOmZKSEnzzzTfw8PDA559/jkWLFkGhUBis387ODvHx8Xj55Zd1jl+4cAFpaWmws7OT7N6sJS0tDe+//z4mTZoEBweHB3rt2bNn4+bNmxgzZgw6d+6M3377DatXr8bu3buRlZUFtVot5s3IyICvry/Gjx+P1q1bIzs7G//+97+xZ88eZGVloWXLlg+07URywMCCJHH+/HmMHz8eHTp0QEpKClxdXXXOL168GP/6179gY2O606ysrOyB/o//nTt3MHr0aBQWFiI1NRUDBgzQOf/RRx9h8eLFeuV27NiB6upqbNiwAUOHDsWhQ4cwaNAgg9cYMWIEvv76a1y/fh3t2rUTj8fHx8PFxQWdO3fG77//bt0be4QsX74cAwYM0PnZCQkJwaBBg7B69Wp8+OGH4vEdO3bolQ8ICMBLL72Eb775BuPHj38gbSaSEw6FkCT+8Y9/oKysDBs3btQLKgCgadOmeP311+Hu7i4emzRpElq1aoXz589jxIgRaN26NSIiIgAA33//PcaMGYP27dtDqVTC3d0ds2bNwh9//KFX965du9C9e3fY2dmhe/fu+PLLL+vc7h07duDkyZP429/+phdUAIBKpcJHH32kd3zr1q149tlnMWTIEHh7e2Pr1q1GrzFq1CgolUokJibqHI+Pj8fYsWPRpEmTOrd33bp16NSpE5o3b46+ffvi+++/N5ivoqIC8+fPxxNPPCE+v7fffhsVFRU6+WrmR2zduhVdunSBnZ0d/Pz8cOjQITHPggUL8NZbbwEAOnbsKA7vXLhwQaeumv8OSqUS3bp1Q1JSUp3vy5SBAwfqBaQDBw6Eo6MjsrOzay1fM9emIYZxiOSAPRYkid27d+OJJ56Av7+/WeXu3LmD4OBgDBgwAEuXLkWLFi0AAImJibh9+zZee+01tG3bFseOHcOqVatw6dIlnS/o7777DmFhYejatStiY2Nx48YNTJ48GY8//nidrv/1118DAP785z/Xuc1XrlzBgQMHsHnzZgBAeHg4VqxYgdWrV8PW1lYvf4sWLTBq1Ch8/vnneO211wAAJ0+exOnTp/Hpp5/Wee7J+vXr8eqrr6J///6YOXMmfvvtN7zwwgtwdHTUCdi0Wi1eeOEF/PDDD4iKioK3tzd+/vlnrFixAr/++it27dqlU+/Bgwexbds2vP7661AqlfjXv/6FkJAQHDt2DN27d8fo0aPx66+/4vPPP8eKFSvEXhcnJyexjh9++AE7d+7EX//6V7Ru3Rr//Oc/ERYWhry8PLRt2xYAUFVVheLi4jrdq6Ojo8nerdLSUpSWlur0ANUQBAE3btzAnTt3cPbsWcyZMwdNmjRp0AmuRI80gcjKiouLBQBCaGio3rnff/9duHbtmphu374tnps4caIAQJgzZ45euXvz1YiNjRUUCoVw8eJF8Zivr6/g6uoqFBUVice+++47AYDQoUOHWtveq1cvwd7evtZ891q6dKnQvHlzoaSkRBAEQfj1118FAMKXX36pk+/AgQMCACExMVHYvXu3oFAohLy8PEEQBOGtt94SPD09BUEQhEGDBgndunUzec3KykrB2dlZ8PX1FSoqKsTj69atEwAIgwYNEo/95z//EWxsbITvv/9ep464uDgBgHD48GHxGAABgHD8+HHx2MWLFwU7OzvhxRdfFI8tWbJEACDk5ubqtQ2AYGtrK5w7d048dvLkSQGAsGrVKr3nUZdk6Dr3WrhwoQBASElJ0TtXUFCgU9fjjz8ubNu2zWR9RFR/7LEgqyspKQEAtGrVSu/c4MGDcfLkSfHzkiVL8Oabb+rkqfkr/l7NmzcX/11WVoY//vgD/fv3hyAIOHHiBNq3b4+CggJkZWVhzpw5sLe3F/M/++yz6Nq1K8rKyurU9tatW9d+k/fYunUrRo4cKZbr3Lkz/Pz8sHXrVoSGhhosM2zYMDg6OiIhIQFvvvkmEhISMGHChDpf8/jx47h69So++OADnV6RSZMmicMUNRITE+Ht7Q0vLy9cv35dPD506FAAwIEDB9C/f3/xeEBAAPz8/MTP7du3x6hRo/DNN9+gurq6TkM1QUFB6NSpk/i5Z8+eUKlU+O2338RjPj4+SE5OrtP93jsh836HDh3C+++/j7Fjx4r3dC9HR0ckJyejvLwcJ06cwM6dO1FaWlqn6xKR+RhYkNXVfMEa+h/vTz75BLdu3UJhYaHeqgjg7twLQ8MWeXl5mDdvHr7++mu9iY013ekXL14EcPeL/X5dunRBZmam+PnatWuorq4WP7dq1QqtWrXS+/KrTXZ2Nk6cOIEJEybg3Llz4vHBgwdjzZo1KCkpgUql0ivXrFkzjBkzBvHx8ejbty/y8/Pxpz/9qc7XNXavzZo1g6enp86xs2fPIjs7W2eo4l5Xr17V+Wzo+T355JO4ffs2rl27ZvJLvkb79u31jrVp00bnv12bNm0QFBRUa12mnDlzBi+++CK6d++OTz/91GAeW1tb8TrPPfccAgMD8fTTT8PZ2RnPPfecRdcnIn0MLMjq7O3t4erqilOnTumdq5lzcf9EvxpKpVJvLL26uhrPPvssbt68idmzZ8PLywstW7bE5cuXMWnSJGi1WrPb+NRTT4lfzgAwf/58LFiwAF5eXjhx4gTy8/N15ikYU7MfxaxZszBr1iy98zt27MDkyZMNlv3Tn/6EuLg4LFiwAD4+PujatavZ91EXWq0WPXr0wPLlyw2er8t9mstYr4YgCOK/KysrcfPmzTrV5+TkpFdnfn4+hg0bBnt7e+zdu7fOPU39+/eHq6srtm7dysCCSAIMLEgSI0eOxKeffopjx46hb9++FtX1888/49dff8XmzZt1hgvu70av2Q/j7NmzenXk5OTofN66davOipKav/Kff/55fP7559iyZQvmzp1rsl2CICA+Ph5DhgzBX//6V73zCxcuxNatW40GFgMGDED79u2RmppqcAmrKffe673d/1VVVcjNzYWPj494rFOnTjh58iQCAwON7q1xL0PP79dff0WLFi3EXo+61FObtLQ0DBkypE55c3NzdXZOvXHjBoYNG4aKigqDy5lrU15eXueJo0RkHgYWJIm3334b8fHxmDJlClJSUuDi4qJz/t6/XGtT85fqvWUEQcDHH3+sk8/V1RW+vr7YvHmzzjyL5ORk/PLLLzobcT399NMGr/XSSy8hNjYWH330EQYPHoyAgACd87du3cKiRYvw0Ucf4fDhw7hw4QI++OADvPTSS3p1/frrr3jvvfdw5coVuLm56Z1XKBT45z//iRMnTpi1CgUA+vTpAycnJ8TFxWHy5MniPItNmzbpLaMcO3Ys9u7di3//+9+IiorSOffHH39Aq9Xq7BWSnp6OzMxM9O7dG8DdnoGvvvoKISEh4n+LmvyWLNms7xyLsrIyjBgxApcvX8aBAwcMDt3U5FMoFOLKoho7duzA77//jj59+tS77URkHAMLkkTnzp0RHx+P8PBwdOnSRdx5UxAE5ObmIj4+HjY2NnVaBurl5YVOnTrhzTffxOXLl6FSqcQvh/vFxsZi5MiRGDBgAKZMmYKbN29i1apV6NatW50m7DVr1gw7d+5EUFAQBg4ciLFjx+Lpp59Gs2bNcPr0acTHx6NNmzb46KOPsHXrVjRp0gQjR440WNcLL7yAv/3tb0hISEBMTIzBPKNGjcKoUaNqbZehdn744Yd49dVXMXToUIwbNw65ubnYuHGj3hyLP//5z9i+fTv+8pe/4MCBA3j66adRXV2NM2fOYPv27fj22291vmS7d++O4OBgneWmAPD++++LeWomd/7tb3/D+PHj0axZMzz//PNmbWZW3zkWEREROHbsGKZMmYLs7GydvStatWolTpg9e/YsgoKCMG7cOHh5ecHGxgbHjx/Hli1b4OHhgRkzZph9bSKqgwZckUIycO7cOeG1114TnnjiCcHOzk5o3ry54OXlJfzlL38RsrKydPJOnDhRaNmypcF6fvnlFyEoKEho1aqV0K5dO2Hq1KniEsaNGzfq5N2xY4fg7e0tKJVKoWvXrsLOnTuFiRMn1mm5aY3ff/9dmDdvntCjRw+hRYsWgp2dndC9e3dh7ty5QkFBgVBZWSm0bdtWeOaZZ0zW07FjR6FXr16CIOguNzWlLstNa/zrX/8SOnbsKCiVSqFPnz7CoUOHhEGDBuksNxWEu8tTFy9eLHTr1k1QKpVCmzZtBD8/P+H9998XiouLxXwAhGnTpglbtmwROnfuLCiVSqFXr17CgQMH9K69cOFC4bHHHhNsbGx0loTW1HG/Dh06CBMnTqzTfZnSoUMHo8tS7/1vfO3aNSEqKkrw8vISWrZsKdja2gqdO3cWZs6cKVy7ds3idhCRYQpBMKNPmogeaQqFAtOmTcPq1asbuilE9JDilt5ERERkNQwsiIiIyGoYWBAREZHVMLAgIpEgCJxfQSSRNWvWwMPDA3Z2dvD398exY8eM5j19+jTCwsLg4eEBhUKBlStX1qvO8vJyTJs2DW3btkWrVq0QFhaGwsJCa96WHgYWREREEtu2bRtiYmIwf/58ZGZmwsfHB8HBwXpb6te4ffs2PD09sWjRIqPb6NelzlmzZuGbb75BYmIiDh48iCtXrmD06NGS3GMNrgohIiKSmL+/P5566imxR1Cr1cLd3R3Tp0/HnDlzTJb18PDAzJkzMXPmTLPqLC4uhpOTE+Lj48VN/M6cOQNvb2+kp6ejX79+1r9RPCIbZGm1Wly5cgWtW7e2ylbDRET0YAmCgFu3bsHNzU3vfUHWVF5ejsrKSovrEQRB7/tGqVRCqVTq5a2srERGRobOawJsbGwQFBSE9PT0el2/LnVmZGSgqqpKZyM6Ly8vtG/fnoFFba5cuSLJi5SIiOjBys/Pr9OOvPVRXl6Ojh1aQXO1uvbMtWjVqpXebr41LzO83/Xr11FdXa33agMXFxecOXOmXtevS50ajQa2trZwcHDQy6PRaOp13bp4JAKLmrcaPr7gXdjY2TVwa4iIyFza8nJcWvBhnd9SWx+VlZXQXK1GbkYHqFrXv1ek5JYWHf0uIj8/HyqVSjxuqLdCjh6JwKKmO8rGzo6BBRHRQ+xBDGerWttYFFiI9ahUOoGFMe3atUOTJk30VmMUFhYanZhpjTrVajUqKytRVFSk02thyXXrgqtCiIhIVqoFrcXJHLa2tvDz80NKSop4TKvVIiUlRe8Nytas08/PD82aNdPJk5OTg7y8vHpfty4eiR4LIiKiutJCgBb1XxBZn7IxMTGYOHEi+vTpg759+2LlypUoKyvD5MmTAQATJkzAY489htjYWAB3h21++eUX8d+XL19GVlYWWrVqhSeeeKJOddrb2yMyMhIxMTFwdHSESqXC9OnTERAQINnETUDCHgtzNgIBgMTERHh5ecHOzg49evTA3r17pWoaERHJmNYK/2eucePGYenSpZg3bx58fX2RlZWFpKQkcfJlXl4eCgoKxPxXrlxBr1690KtXLxQUFGDp0qXo1asXXnnllTrXCQArVqzAc889h7CwMAwcOBBqtRo7d+604OnVTpJ9LLZt24YJEyYgLi4O/v7+WLlyJRITE5GTkwNnZ2e9/GlpaRg4cCBiY2Px3HPPIT4+HosXL0ZmZia6d+9e6/VKSkpgb2+P9os+5BwLIqKHkLa8HHlz3kVxcXGd5i3UR813xZWcxy2evOnW5ZKkbX2YSdJjsXz5ckydOhWTJ09G165dERcXhxYtWmDDhg0G83/88ccICQnBW2+9BW9vbyxcuBC9e/fm1sJERGR11YJgcSLjrB5Y1Gzace+GHLVtBJKenq6THwCCg4ON5q+oqEBJSYlOIiIiqouaORaWJDLO6oGFqU07jG3IodFozMofGxsLe3t7MXFzLCIiosbhoVxuOnfuXBQXF4spPz+/oZtEREQPCS0EVFuQ2GNhmtWXm9ZnIxC1Wm1WfmP7sRMREdWmIZabyonVeyzqsxFIQECATn4ASE5OlnQDDyIiIrI+STbIMncjkBkzZmDQoEFYtmwZRo4ciYSEBBw/fhzr1q2TonlERCRjlq7s4KoQ0yQJLMaNG4dr165h3rx50Gg08PX11dsI5N7X4vbv3x/x8fF499138c4776Bz587YtWtXnfawICIiMof2f8mS8mScJBtkPWjcIIuI6OH2IDfIOpPtgtYWbJB165YWXt6F3CDLCL4rhIiIZKVmdYcl5ck4BhZERCQr1cLdZEl5Mo6BBRERyQrnWEjrodwgi4iIiBon9lgQEZGsaKFANRQWlSfjGFgQEZGsaIW7yZLyZByHQoiIiMhq2GNBRESyUm3hUIglZeWAgQUREckKAwtpcSiEiIiIrIY9FkREJCtaQQGtYMGqEAvKygEDCyIikhUOhUiLQyFERERkNeyxICIiWamGDaot+Lu62opteRQxsCAiIlkRLJxjIXCOhUkMLIiISFY4x0JanGNBREREVsMeCyIikpVqwQbVggVzLPiuEJMYWBARkaxooYDWgg57LRhZmMKhECIiIrIa9lgQEZGscPKmtBhYEBGRrFg+x4JDIaZwKISIiIishj0WREQkK3cnb1rwEjIOhZjEwIKIiGRFa+GW3lwVYhqHQoiIiMhqGFgQEZGs1EzetCTVx5o1a+Dh4QE7Ozv4+/vj2LFjJvMnJibCy8sLdnZ26NGjB/bu3atzXqFQGExLliwR83h4eOidX7RoUb3aX1cMLIiISFa0sLE4mWvbtm2IiYnB/PnzkZmZCR8fHwQHB+Pq1asG86elpSE8PByRkZE4ceIEQkNDERoailOnTol5CgoKdNKGDRugUCgQFhamU9cHH3ygk2/69Olmt98cDCyIiEhWqgWFxclcy5cvx9SpUzF58mR07doVcXFxaNGiBTZs2GAw/8cff4yQkBC89dZb8Pb2xsKFC9G7d2+sXr1azKNWq3XSV199hSFDhsDT01OnrtatW+vka9mypdntNwcDCyIionooKSnRSRUVFQbzVVZWIiMjA0FBQeIxGxsbBAUFIT093WCZ9PR0nfwAEBwcbDR/YWEh9uzZg8jISL1zixYtQtu2bdGrVy8sWbIEd+7cqest1gtXhRARkaxUW7gqpPp/q0Lc3d11js+fPx8LFizQy3/9+nVUV1fDxcVF57iLiwvOnDlj8BoajcZgfo1GYzD/5s2b0bp1a4wePVrn+Ouvv47evXvD0dERaWlpmDt3LgoKCrB8+XKT92gJBhZERCQrWsEGWgt23tT+b+fN/Px8qFQq8bhSqbS4bfW1YcMGREREwM7OTud4TEyM+O+ePXvC1tYWr776KmJjYyVrLwMLIiKielCpVDqBhTHt2rVDkyZNUFhYqHO8sLAQarXaYBm1Wl3n/N9//z1ycnKwbdu2Wtvi7++PO3fu4MKFC+jSpUut+euDcyyIiEhWaoZCLEnmsLW1hZ+fH1JSUsRjWq0WKSkpCAgIMFgmICBAJz8AJCcnG8y/fv16+Pn5wcfHp9a2ZGVlwcbGBs7OzmbdgznYY0FERLKiBeq1suPe8uaKiYnBxIkT0adPH/Tt2xcrV65EWVkZJk+eDACYMGECHnvsMcTGxgIAZsyYgUGDBmHZsmUYOXIkEhIScPz4caxbt06n3pKSEiQmJmLZsmV610xPT8fRo0cxZMgQtG7dGunp6Zg1axZefvlltGnTph53UTdW77GIjY3FU089hdatW8PZ2RmhoaHIyckxWWbTpk16G3jcP05ERET0sBo3bhyWLl2KefPmwdfXF1lZWUhKShInaObl5aGgoEDM379/f8THx2PdunXw8fHBF198gV27dqF79+469SYkJEAQBISHh+tdU6lUIiEhAYMGDUK3bt3w0UcfYdasWXrBibUpBMG6738NCQnB+PHj8dRTT+HOnTt45513cOrUKfzyyy9G185u2rQJM2bM0AlAFAqF3oxYY0pKSmBvb4/2iz6EDQMSIqKHjra8HHlz3kVxcXGd5i3UR813xdrMp9C8Vf077P8ovYPXev8oaVsfZlYfCklKStL5vGnTJjg7OyMjIwMDBw40Wk6hUBidxEJERGQtlmzLXVOejJP86RQXFwMAHB0dTeYrLS1Fhw4d4O7ujlGjRuH06dNG81ZUVOhtTEJEREQNT9LAQqvVYubMmXj66af1xoXu1aVLF2zYsAFfffUVtmzZAq1Wi/79++PSpUsG88fGxsLe3l5M929SQkREZIwWCosTGSdpYDFt2jScOnUKCQkJJvMFBARgwoQJ8PX1xaBBg7Bz5044OTnhk08+MZh/7ty5KC4uFlN+fr4UzSciokdQQ73dVC4kW24aHR2N3bt349ChQ3j88cfNKtusWTP06tUL586dM3heqVQ26A5nRET08LJ8S28GFqZY/ekIgoDo6Gh8+eWX2L9/Pzp27Gh2HdXV1fj555/h6upq7eYRERGRhKzeYzFt2jTEx8fjq6++QuvWrcUXptjb26N58+YA9DcC+eCDD9CvXz888cQTKCoqwpIlS3Dx4kW88sor1m4eERHJnFZQQGvJBlkWlJUDqwcWa9euBQAMHjxY5/jGjRsxadIkAHc3ArGx+f/Okt9//x1Tp06FRqNBmzZt4Ofnh7S0NHTt2tXazSMiIpnTWjgUouVQiElWDyzqst9WamqqzucVK1ZgxYoV1m4KERERPWB8VwgREcmK5a9NZ4+FKQwsiIhIVqqhQLUFe1FYUlYOGHYRERGR1bDHgoiIZIVDIdJiYEFERLJSDcuGM6qt15RHEsMuIiIishr2WBARkaxwKERaDCyIiEhWLH2RGF9CZhoDCyIikhXBwlefC1xuahLDLiIiIrIa9lgQEZGscChEWgwsiIhIVvh2U2kx7CIiIiKrYY8FERHJSrWFr023pKwcMLAgIiJZ4VCItBh2ERERkdWwx4KIiGRFCxtoLfi72pKycsDAgoiIZKVaUKDaguEMS8rKAcMuIiIishr2WBARkaxw8qa0GFgQEZGsCBa+3VTgzpsmMbAgIiJZqYYC1Ra8SMySsnLAsIuIiIishj0WREQkK1rBsnkSWsGKjXkEMbAgIiJZ0Vo4x8KSsnLAp0NERERWw8CCiIhkRQuFxak+1qxZAw8PD9jZ2cHf3x/Hjh0zmT8xMRFeXl6ws7NDjx49sHfvXp3zkyZNgkKh0EkhISE6eW7evImIiAioVCo4ODggMjISpaWl9Wp/XTGwICIiWanZedOSZK5t27YhJiYG8+fPR2ZmJnx8fBAcHIyrV68azJ+Wlobw8HBERkbixIkTCA0NRWhoKE6dOqWTLyQkBAUFBWL6/PPPdc5HRETg9OnTSE5Oxu7du3Ho0CFERUWZ3X5zMLAgIiKS2PLlyzF16lRMnjwZXbt2RVxcHFq0aIENGzYYzP/xxx8jJCQEb731Fry9vbFw4UL07t0bq1ev1smnVCqhVqvF1KZNG/FcdnY2kpKS8Omnn8Lf3x8DBgzAqlWrkJCQgCtXrkh2rwwsiIhIVmomb1qSAKCkpEQnVVRUGLxeZWUlMjIyEBQUJB6zsbFBUFAQ0tPTDZZJT0/XyQ8AwcHBevlTU1Ph7OyMLl264LXXXsONGzd06nBwcECfPn3EY0FBQbCxscHRo0fNe2hmYGBBRESyooVC3Na7Xul/cyzc3d1hb28vptjYWIPXu379Oqqrq+Hi4qJz3MXFBRqNxmAZjUZTa/6QkBB89tlnSElJweLFi3Hw4EEMHz4c1dXVYh3Ozs46dTRt2hSOjo5Gr2sNXG5KRERUD/n5+VCpVOJnpVL5QK8/fvx48d89evRAz5490alTJ6SmpiIwMPCBtuVe7LEgIiJZESxcESL8r8dCpVLpJGOBRbt27dCkSRMUFhbqHC8sLIRarTZYRq1Wm5UfADw9PdGuXTucO3dOrOP+yaF37tzBzZs3TdZjKQYWREQkKxYNg9Tjzai2trbw8/NDSkrK/7dBq0VKSgoCAgIMlgkICNDJDwDJyclG8wPApUuXcOPGDbi6uop1FBUVISMjQ8yzf/9+aLVa+Pv7m3UP5uBQCBERyUpD7LwZExODiRMnok+fPujbty9WrlyJsrIyTJ48GQAwYcIEPPbYY+I8jRkzZmDQoEFYtmwZRo4ciYSEBBw/fhzr1q0DAJSWluL9999HWFgY1Go1zp8/j7fffhtPPPEEgoODAQDe3t4ICQnB1KlTERcXh6qqKkRHR2P8+PFwc3Or9/3Xxuo9FgsWLNDbsMPLy8tkmdo2ASEiInqYjRs3DkuXLsW8efPg6+uLrKwsJCUliRM08/LyUFBQIObv378/4uPjsW7dOvj4+OCLL77Arl270L17dwBAkyZN8NNPP+GFF17Ak08+icjISPj5+eH777/XGZLZunUrvLy8EBgYiBEjRmDAgAFicCIVSXosunXrhn379v3/RZoav0zNJiCxsbF47rnnEB8fj9DQUGRmZooPkIiIyFrqM5xxf/n6iI6ORnR0tMFzqampesfGjBmDMWPGGMzfvHlzfPvtt7Ve09HREfHx8Wa101KSzLFo2rSpzoYd7dq1M5q3rpuAEBERWUNDbektF5IEFmfPnoWbmxs8PT0RERGBvLw8o3nrugnIvSoqKvQ2JiEiIqKGZ/XAwt/fH5s2bUJSUhLWrl2L3NxcPPPMM7h165bB/HXZBOR+sbGxOpuSuLu7W/UeiIjo0fWgV4XIjdUDi+HDh2PMmDHo2bMngoODsXfvXhQVFWH79u1Wu8bcuXNRXFwspvz8fKvVTUREjzYGFtKSfLmpg4MDnnzySXHDjvvVZxMQpVL5wHc4IyIiotpJvkFWaWkpzp8/L27Ycb/6bAJCRERUX+yxkJbVA4s333wTBw8exIULF5CWloYXX3wRTZo0QXh4OIC7m4DMnTtXzD9jxgwkJSVh2bJlOHPmDBYsWIDjx48bXZJDRERkCQYW0rL6UMilS5cQHh6OGzduwMnJCQMGDMCRI0fg5OQE4O4mIDY2/x/P1GwC8u677+Kdd95B586ddTYBISIiooeH1QOLhIQEk+fN3QSEiIjImgTAor0oBOs15ZHEd4UQEZGsNNTOm3LBwIKIiGSFgYW0+Np0IiIishr2WBARkaywx0JaDCyIiEhWGFhIi0MhREREZDXssSAiIlkRBAUEC3odLCkrBwwsiIhIVrRQWLSPhSVl5YBDIURERGQ17LEgIiJZ4eRNaTGwICIiWeEcC2lxKISIiIishj0WREQkKxwKkRYDCyIikhUOhUiLgQUREcmKYGGPBQML0zjHgoiIiKyGPRZERCQrAgBBsKw8GcfAgoiIZEULBRTceVMyHAohIiIiq2GPBRERyQpXhUiLgQUREcmKVlBAwX0sJMOhECIiIrIa9lgQEZGsCIKFq0K4LMQkBhZERCQrnGMhLQ6FEBERkdWwx4KIiGSFPRbSYo8FERHJSs3bTS1J9bFmzRp4eHjAzs4O/v7+OHbsmMn8iYmJ8PLygp2dHXr06IG9e/eK56qqqjB79mz06NEDLVu2hJubGyZMmIArV67o1OHh4QGFQqGTFi1aVK/21xUDCyIikpWayZuWJHNt27YNMTExmD9/PjIzM+Hj44Pg4GBcvXrVYP60tDSEh4cjMjISJ06cQGhoKEJDQ3Hq1CkAwO3bt5GZmYn33nsPmZmZ2LlzJ3JycvDCCy/o1fXBBx+goKBATNOnTzf/BszAwIKIiEhiy5cvx9SpUzF58mR07doVcXFxaNGiBTZs2GAw/8cff4yQkBC89dZb8Pb2xsKFC9G7d2+sXr0aAGBvb4/k5GSMHTsWXbp0Qb9+/bB69WpkZGQgLy9Pp67WrVtDrVaLqWXLlpLeKwMLIiKSlbu9DgoL0t16SkpKdFJFRYXB61VWViIjIwNBQUHiMRsbGwQFBSE9Pd1gmfT0dJ38ABAcHGw0PwAUFxdDoVDAwcFB5/iiRYvQtm1b9OrVC0uWLMGdO3fq8JTqj5M3iYhIVqw1edPd3V3n+Pz587FgwQK9/NevX0d1dTVcXFx0jru4uODMmTMGr6HRaAzm12g0BvOXl5dj9uzZCA8Ph0qlEo+//vrr6N27NxwdHZGWloa5c+eioKAAy5cvr/U+64uBBRERUT3k5+frfIkrlcoGaUdVVRXGjh0LQRCwdu1anXMxMTHiv3v27AlbW1u8+uqriI2Nlay9DCyIiEhWhP8lS8oDgEql0gksjGnXrh2aNGmCwsJCneOFhYVQq9UGy6jV6jrlrwkqLl68iP3799faHn9/f9y5cwcXLlxAly5dam17fXCOBRERyYpl8yvMH0axtbWFn58fUlJSxGNarRYpKSkICAgwWCYgIEAnPwAkJyfr5K8JKs6ePYt9+/ahbdu2tbYlKysLNjY2cHZ2NusezMEeCyIiIonFxMRg4sSJ6NOnD/r27YuVK1eirKwMkydPBgBMmDABjz32GGJjYwEAM2bMwKBBg7Bs2TKMHDkSCQkJOH78ONatWwfgblDx0ksvITMzE7t370Z1dbU4/8LR0RG2trZIT0/H0aNHMWTIELRu3Rrp6emYNWsWXn75ZbRp00aye2VgQURE8mKtsRAzjBs3DteuXcO8efOg0Wjg6+uLpKQkcYJmXl4ebGz+fxChf//+iI+Px7vvvot33nkHnTt3xq5du9C9e3cAwOXLl/H1118DAHx9fXWudeDAAQwePBhKpRIJCQlYsGABKioq0LFjR8yaNUtn3oUUFIJg3fe0eXh44OLFi3rH//rXv2LNmjV6xzdt2iRGbDWUSiXKy8vrfM2SkhLY29uj/aIPYWNnZ36jiYioQWnLy5E3510UFxfXad5CfdR8V3hu+htsWtT/u0J7uxy/TfpI0rY+zKzeY/Hjjz+iurpa/Hzq1Ck8++yzGDNmjNEyKpUKOTk54meFgvuwExGRNPjadGlZPbBwcnLS+bxo0SJ06tQJgwYNMlpGoVAYnRlLREREDw9JV4VUVlZiy5YtmDJlisleiNLSUnTo0AHu7u4YNWoUTp8+bbLeiooKvR3PiIiI6uJBrwqRG0kDi127dqGoqAiTJk0ymqdLly7YsGEDvvrqK2zZsgVarRb9+/fHpUuXjJaJjY2Fvb29mO7f/YyIiMgoQWF5IqMkDSzWr1+P4cOHw83NzWiegIAATJgwAb6+vhg0aBB27twJJycnfPLJJ0bLzJ07F8XFxWLKz8+XovlERERkJsmWm168eBH79u3Dzp07zSrXrFkz9OrVC+fOnTOaR6lUNtjWqURE9HDj5E1pSdZjsXHjRjg7O2PkyJFmlauursbPP/8MV1dXiVpGRESyJlghkVGSBBZarRYbN27ExIkT0bSpbqfIhAkTMHfuXPHzBx98gO+++w6//fYbMjMz8fLLL+PixYt45ZVXpGgaERERSUiSoZB9+/YhLy8PU6ZM0Tt3/+5iv//+O6ZOnQqNRoM2bdrAz88PaWlp6Nq1qxRNIyIimbPWa9PJMEkCi2HDhsHYhp6pqak6n1esWIEVK1ZI0QwiIiLDOJwhGb7dlIiIiKyGLyEjIiJZ4VCItBhYEBGRvDTA203lhIEFERHJjOJ/yZLyZAznWBAREZHVsMeCiIjkhUMhkmJgQURE8sLAQlIcCiEiIiKrYY8FERHJi6WvPudyU5MYWBARkazw7abS4lAIERERWQ17LIiISF44eVNSDCyIiEheOMdCUhwKISIiIqthjwUREcmKQribLClPxjGwICIieeEcC0kxsCAiInnhHAtJcY4FERERWQ17LIiISF44FCIpBhZERCQvDCwkxaEQIiIishr2WBARkbywx0JSDCyIiEheuCpEUhwKISIiIqthjwUREckKd96UFgMLIiKSF86xkBSHQoiIiB6ANWvWwMPDA3Z2dvD398exY8dM5k9MTISXlxfs7OzQo0cP7N27V+e8IAiYN28eXF1d0bx5cwQFBeHs2bM6eW7evImIiAioVCo4ODggMjISpaWlVr+3ezGwICIikti2bdsQExOD+fPnIzMzEz4+PggODsbVq1cN5k9LS0N4eDgiIyNx4sQJhIaGIjQ0FKdOnRLz/OMf/8A///lPxMXF4ejRo2jZsiWCg4NRXl4u5omIiMDp06eRnJyM3bt349ChQ4iKipL0XhWCIDz0nTolJSWwt7dH+0UfwsbOrqGbQ0REZtKWlyNvzrsoLi6GSqWS5Bo13xUdFlv2XaEtL8fF2e8iPz9fp61KpRJKpdJgGX9/fzz11FNYvXr13Tq0Wri7u2P69OmYM2eOXv5x48ahrKwMu3fvFo/169cPvr6+iIuLgyAIcHNzwxtvvIE333wTAFBcXAwXFxds2rQJ48ePR3Z2Nrp27Yoff/wRffr0AQAkJSVhxIgRuHTpEtzc3Or9DExhjwUREclLzXJTSxIAd3d32Nvbiyk2Ntbg5SorK5GRkYGgoCDxmI2NDYKCgpCenm6wTHp6uk5+AAgODhbz5+bmQqPR6OSxt7eHv7+/mCc9PR0ODg5iUAEAQUFBsLGxwdGjR+vx4OqGkzeJiIjqwVCPhSHXr19HdXU1XFxcdI67uLjgzJkzBstoNBqD+TUajXi+5pipPM7OzjrnmzZtCkdHRzGPFBhYEBGRvFhpVYhKpZJs2OZhxqEQIiKSF8EKyQzt2rVDkyZNUFhYqHO8sLAQarXaYBm1Wm0yf83/ry3P/ZND79y5g5s3bxq9rjUwsCAiIpKQra0t/Pz8kJKSIh7TarVISUlBQECAwTIBAQE6+QEgOTlZzN+xY0eo1WqdPCUlJTh69KiYJyAgAEVFRcjIyBDz7N+/H1qtFv7+/la7v/txKISIiGSlIXbejImJwcSJE9GnTx/07dsXK1euRFlZGSZPngwAmDBhAh577DFxAuiMGTMwaNAgLFu2DCNHjkRCQgKOHz+OdevW3W2DQoGZM2fiww8/ROfOndGxY0e89957cHNzQ2hoKADA29sbISEhmDp1KuLi4lBVVYXo6GiMHz9eshUhQD16LA4dOoTnn38ebm5uUCgU2LVrl875umzYYYi5G4cQERHVywMeCgHuLh9dunQp5s2bB19fX2RlZSEpKUmcfJmXl4eCggIxf//+/REfH49169bBx8cHX3zxBXbt2oXu3buLed5++21Mnz4dUVFReOqpp1BaWoqkpCTY3bOUduvWrfDy8kJgYCBGjBiBAQMGiMGJVMzex+K///0vDh8+DD8/P4wePRpffvmlGB0BwOLFixEbG4vNmzeLEdTPP/+MX375Redm77Vt2zZMmDABcXFx8Pf3x8qVK5GYmIicnBy9Ga2GcB8LIqKH24Pcx8Ljw48s3sfiwrt/k7StDzOzeyyGDx+ODz/8EC+++KLeOUEQsHLlSrz77rsYNWoUevbsic8++wxXrlzR69m41/LlyzF16lRMnjwZXbt2RVxcHFq0aIENGzaY2zwiIiLTGqDHQk6sOnmzLht23K8+G4dUVFSgpKREJxEREdVFzRwLSxIZZ9XAoi4bdtzP1MYhxsrExsbq7Hbm7u5uhdYTERGRpR7K5aZz585FcXGxmPLz8xu6SURE9LCw0pbeZJhVl5veu2GHq6ureLywsBC+vr4Gy9Rn4xBTL3ohIiIyyUo7b5JhVu2xqMuGHferz8YhRERE9cU5FtIyu8eitLQU586dEz/n5uYiKysLjo6OaN++fa0bdgBAYGAgXnzxRURHRwOofeMQIiIiejiYHVgcP34cQ4YMET/HxMQAACZOnIhNmzbh7bffRllZGaKiolBUVIQBAwbobdhx/vx5XL9+Xfw8btw4XLt2DfPmzYNGo4Gvr6/OxiFERERWw6EQSZm9QVZjxA2yiIgebg9ygyzP9/6OJhZ8V1SXl+O3he9wgywjHspVIURERNQ48SVkREQkLxwKkRQDCyIikhcGFpLiUAgRERFZDXssiIhIVizdi4L7WJjGHgsiIiKyGgYWREREZDUcCiEiInnh5E1JMbAgIiJZ4RwLaTGwICIi+WFwIBnOsSAiIiKrYY8FERHJC+dYSIqBBRERyQrnWEiLQyFERERkNeyxICIieeFQiKQYWBARkaxwKERaHAohIiIiq2GPBRERyQuHQiTFwIKIiOSFgYWkOBRCREREVsMeCyIikhVO3pQWAwsiIpIXDoVIioEFERHJCwMLSXGOBREREVkNeyyIiEhWOMdCWgwsiIhIXjgUIikOhRARETUiN2/eREREBFQqFRwcHBAZGYnS0lKTZcrLyzFt2jS0bdsWrVq1QlhYGAoLC8XzJ0+eRHh4ONzd3dG8eXN4e3vj448/1qkjNTUVCoVCL2k0GrPazx4LIiKSlcY+FBIREYGCggIkJyejqqoKkydPRlRUFOLj442WmTVrFvbs2YPExETY29sjOjoao0ePxuHDhwEAGRkZcHZ2xpYtW+Du7o60tDRERUWhSZMmiI6O1qkrJycHKpVK/Ozs7GxW+xlYEBGRvDTioZDs7GwkJSXhxx9/RJ8+fQAAq1atwogRI7B06VK4ubnplSkuLsb69esRHx+PoUOHAgA2btwIb29vHDlyBP369cOUKVN0ynh6eiI9PR07d+7UCyycnZ3h4OBQ73vgUAgREVE9lJSU6KSKigqL60xPT4eDg4MYVABAUFAQbGxscPToUYNlMjIyUFVVhaCgIPGYl5cX2rdvj/T0dKPXKi4uhqOjo95xX19fuLq64tlnnxV7PMzBwIKIiORFsEIC4O7uDnt7ezHFxsZa3DSNRqM39NC0aVM4Ojoaneug0Whga2ur18vg4uJitExaWhq2bduGqKgo8Zirqyvi4uKwY8cO7NixA+7u7hg8eDAyMzPNugcOhRARkawo/pcsKQ8A+fn5OnMRlEql0TJz5szB4sWLTdabnZ1tQavq7tSpUxg1ahTmz5+PYcOGice7dOmCLl26iJ/79++P8+fPY8WKFfjPf/5T5/oZWBAREdWDSqXSCSxMeeONNzBp0iSTeTw9PaFWq3H16lWd43fu3MHNmzehVqsNllOr1aisrERRUZFOr0VhYaFemV9++QWBgYGIiorCu+++W2u7+/btix9++KHWfPdiYEFERPLSAJM3nZyc4OTkVGu+gIAAFBUVISMjA35+fgCA/fv3Q6vVwt/f32AZPz8/NGvWDCkpKQgLCwNwd2VHXl4eAgICxHynT5/G0KFDMXHiRHz00Ud1andWVhZcXV3rlLcGAwsiIpKVxrzc1NvbGyEhIZg6dSri4uJQVVWF6OhojB8/XlwRcvnyZQQGBuKzzz5D3759YW9vj8jISMTExMDR0REqlQrTp09HQEAA+vXrB+Du8MfQoUMRHByMmJgYce5FkyZNxIBn5cqV6NixI7p164by8nJ8+umn2L9/P7777juz7sHsyZuHDh3C888/Dzc3NygUCuzatUs8V1VVhdmzZ6NHjx5o2bIl3NzcMGHCBFy5csVknQsWLNDbkMPLy8vcphEREdXOSpM3pbJ161Z4eXkhMDAQI0aMwIABA7Bu3TrxfFVVFXJycnD79m3x2IoVK/Dcc88hLCwMAwcOhFqtxs6dO8XzX3zxBa5du4YtW7bA1dVVTE899ZSYp7KyEm+88QZ69OiBQYMG4eTJk9i3bx8CAwPNar9CEASzHtF///tfHD58GH5+fhg9ejS+/PJLhIaGAri7dOWll17C1KlT4ePjg99//x0zZsxAdXU1jh8/brTOBQsW4IsvvsC+ffvEY02bNkW7du3q1KaSkhLY29uj/aIPYWNnZ87tEBFRI6AtL0fenHdRXFxc53kL5qr5ruj26t/RRFn/74rqinKc/uQdSdv6MDN7KGT48OEYPny4wXP29vZITk7WObZ69Wr07dsXeXl5aN++vfGGNG1qdGIKERGRVfF9H5KRfB+L4uJiKBSKWnfxOnv2LNzc3ODp6YmIiAjk5eUZzVtRUaG3MQkREVFd1MyxsCSRcZIGFuXl5Zg9ezbCw8NNdhf5+/tj06ZNSEpKwtq1a5Gbm4tnnnkGt27dMpg/NjZWZ1MSd3d3qW6BiIiIzCBZYFFVVYWxY8dCEASsXbvWZN7hw4djzJgx6NmzJ4KDg7F3714UFRVh+/btBvPPnTsXxcXFYsrPz5fiFoiI6FHUyCdvPuwkWW5aE1RcvHgR+/fvN3tyi4ODA5588kmcO3fO4HmlUmlyhzMiIiJjGvNy00eB1XssaoKKs2fPYt++fWjbtq3ZdZSWluL8+fNmb8pBREREDcvswKK0tBRZWVnIysoCAOTm5iIrKwt5eXmoqqrCSy+9hOPHj2Pr1q2orq6GRqOBRqNBZWWlWEdgYCBWr14tfn7zzTdx8OBBXLhwAWlpaXjxxRfRpEkThIeHW36HRERE9+JQiKTMHgo5fvw4hgwZIn6OiYkBAEycOBELFizA119/DeDua1fvdeDAAQwePBgAcP78eVy/fl08d+nSJYSHh+PGjRtwcnLCgAEDcOTIkTptf0pERGQODoVIy+zAYvDgwTC1p1Zd9tu6cOGCzueEhARzm0FERESNEN8VQkRE8tIALyGTEwYWREQkLwwsJMXAgoiIZIVzLKQl+ZbeREREJB/ssSAiInnhUIikGFgQEZGsKAQBijqsYDRVnozjUAgRERFZDXssiIhIXjgUIikGFkREJCtcFSItDoUQERGR1bDHgoiI5IVDIZJiYEFERLLCoRBpcSiEiIiIrIY9FkREJC8cCpEUAwsiIpIVDoVIi4EFERHJC3ssJMU5FkRERGQ17LEgIiLZ4XCGdBhYEBGRvAjC3WRJeTKKQyFERERkNeyxICIiWeGqEGkxsCAiInnhqhBJcSiEiIiIrIY9FkREJCsK7d1kSXkyjoEFERHJC4dCJMWhECIiIrIaBhZERCQrNatCLElSunnzJiIiIqBSqeDg4IDIyEiUlpaaLFNeXo5p06ahbdu2aNWqFcLCwlBYWKiTR6FQ6KWEhASdPKmpqejduzeUSiWeeOIJbNq0yez2M7AgIiJ5qdkgy5IkoYiICJw+fRrJycnYvXs3Dh06hKioKJNlZs2ahW+++QaJiYk4ePAgrly5gtGjR+vl27hxIwoKCsQUGhoqnsvNzcXIkSMxZMgQZGVlYebMmXjllVfw7bffmtV+zrEgIiJZacz7WGRnZyMpKQk//vgj+vTpAwBYtWoVRowYgaVLl8LNzU2vTHFxMdavX4/4+HgMHToUwN0AwtvbG0eOHEG/fv3EvA4ODlCr1QavHRcXh44dO2LZsmUAAG9vb/zwww9YsWIFgoOD63wP7LEgIiKqh5KSEp1UUVFhcZ3p6elwcHAQgwoACAoKgo2NDY4ePWqwTEZGBqqqqhAUFCQe8/LyQvv27ZGenq6Td9q0aWjXrh369u2LDRs2QLin9yU9PV2nDgAIDg7Wq6M2DCyIiEheBCskAO7u7rC3txdTbGysxU3TaDRwdnbWOda0aVM4OjpCo9EYLWNrawsHBwed4y4uLjplPvjgA2zfvh3JyckICwvDX//6V6xatUqnHhcXF706SkpK8Mcff9T5HjgUQkREsmKtoZD8/HyoVCrxuFKpNFpmzpw5WLx4scl6s7Oz69+oOnjvvffEf/fq1QtlZWVYsmQJXn/9dateh4EFERFRPahUKp3AwpQ33ngDkyZNMpnH09MTarUaV69e1Tl+584d3Lx50+jcCLVajcrKShQVFen0WhQWFhotAwD+/v5YuHAhKioqoFQqoVar9VaSFBYWQqVSoXnz5qZv8B4MLIiISF4a4LXpTk5OcHJyqjVfQEAAioqKkJGRAT8/PwDA/v37odVq4e/vb7CMn58fmjVrhpSUFISFhQEAcnJykJeXh4CAAKPXysrKQps2bcSeloCAAOzdu1cnT3Jyssk6DGFgQUREstKYV4V4e3sjJCQEU6dORVxcHKqqqhAdHY3x48eLK0IuX76MwMBAfPbZZ+jbty/s7e0RGRmJmJgYODo6QqVSYfr06QgICBBXhHzzzTcoLCxEv379YGdnh+TkZPz973/Hm2++KV77L3/5C1avXo23334bU6ZMwf79+7F9+3bs2bPHrHswe/LmoUOH8Pzzz8PNzQ0KhQK7du3SOT9p0iS9DThCQkJqrXfNmjXw8PCAnZ0d/P39cezYMXObRkRE9NDbunUrvLy8EBgYiBEjRmDAgAFYt26deL6qqgo5OTm4ffu2eGzFihV47rnnEBYWhoEDB0KtVmPnzp3i+WbNmmHNmjUICAiAr68vPvnkEyxfvhzz588X83Ts2BF79uxBcnIyfHx8sGzZMnz66admLTUF6tFjUVZWBh8fH0yZMsXg5hsAEBISgo0bN4qfTU1oAYBt27YhJiYGcXFx8Pf3x8qVKxEcHIycnBy92bFEREQWaeTvCnF0dER8fLzR8x4eHjrLRAHAzs4Oa9aswZo1awyWCQkJqdMf+YMHD8aJEyfMa/B9zA4shg8fjuHDh5vMUzMJpK6WL1+OqVOnYvLkyQDubtKxZ88ebNiwAXPmzDG3iUREREY15qGQR4Ek+1ikpqbC2dkZXbp0wWuvvYYbN24YzVtZWYmMjAydTTlsbGwQFBRkdFOOiooKvY1JiIiIqOFZPbAICQnBZ599hpSUFCxevBgHDx7E8OHDUV1dbTD/9evXUV1dbXBTDmObgcTGxupsSuLu7m7t2yAiokeVVrA8kVFWXxUyfvx48d89evRAz5490alTJ6SmpiIwMNAq15g7dy5iYmLEzyUlJQwuiIiobhr5HIuHneRbent6eqJdu3Y4d+6cwfPt2rVDkyZNDG7KYWyehlKpFDcmMWeDEiIiIgUsfG16Q99AIyd5YHHp0iXcuHEDrq6uBs/b2trCz88PKSkp4jGtVouUlBSzN+UgIiKihmV2YFFaWoqsrCxkZWUBuPv+9qysLOTl5aG0tBRvvfUWjhw5ggsXLiAlJQWjRo3CE088obMONjAwEKtXrxY/x8TE4N///jc2b96M7OxsvPbaaygrKxNXiRAREVlNzc6bliQyyuw5FsePH8eQIUPEzzVzHSZOnIi1a9fip59+wubNm1FUVAQ3NzcMGzYMCxcu1NnL4vz587h+/br4edy4cbh27RrmzZsHjUYDX19fJCUl6U3oJCIishSXm0rL7MBi8ODBehtz3Ovbb7+ttY4LFy7oHYuOjkZ0dLS5zSEiIqJGhO8KISIieeGqEEkxsCAiIllRCAIUFsyTsKSsHEi+KoSIiIjkgz0WREQkL9r/JUvKk1EMLIiISFY4FCItDoUQERGR1bDHgoiI5IWrQiTFwIKIiOTF0t0zORRiEgMLIiKSFe68KS3OsSAiIiKrYY8FERHJC4dCJMXAgoiIZEWhvZssKU/GcSiEiIiIrIY9FkREJC8cCpEUAwsiIpIX7mMhKQ6FEBERkdWwx4KIiGSF7wqRFgMLIiKSF86xkBSHQoiIiMhq2GNBRETyIgCwZC8KdliYxMCCiIhkhXMspMXAgoiI5EWAhXMsrNaSRxLnWBAREZHVsMeCiIjkhatCJMXAgoiI5EULQGFheTKKQyFERERkNQwsiIhIVmpWhViSpHTz5k1ERERApVLBwcEBkZGRKC0tNVmmvLwc06ZNQ9u2bdGqVSuEhYWhsLBQPL9p0yYoFAqD6erVqwCA1NRUg+c1Go1Z7edQCBERyUsjn2MRERGBgoICJCcno6qqCpMnT0ZUVBTi4+ONlpk1axb27NmDxMRE2NvbIzo6GqNHj8bhw4cBAOPGjUNISIhOmUmTJqG8vBzOzs46x3NycqBSqcTP95+vDQMLIiKiRiI7OxtJSUn48ccf0adPHwDAqlWrMGLECCxduhRubm56ZYqLi7F+/XrEx8dj6NChAICNGzfC29sbR44cQb9+/dC8eXM0b95cLHPt2jXs378f69ev16vP2dkZDg4O9b4HDoUQEZG81PRYWJIAlJSU6KSKigqLm5aeng4HBwcxqACAoKAg2NjY4OjRowbLZGRkoKqqCkFBQeIxLy8vtG/fHunp6QbLfPbZZ2jRogVeeuklvXO+vr5wdXXFs88+K/Z4mIOBBRERyYuVAgt3d3fY29uLKTY21uKmaTQavaGHpk2bwtHR0ehcB41GA1tbW71eBhcXF6Nl1q9fjz/96U86vRiurq6Ii4vDjh07sGPHDri7u2Pw4MHIzMw06x44FEJERFQP+fn5OnMRlEql0bxz5szB4sWLTdaXnZ1ttbaZkp6ejuzsbPznP//ROd6lSxd06dJF/Ny/f3+cP38eK1as0MtrCgMLIiKSFyvtY6FSqXQCC1PeeOMNTJo0yWQeT09PqNVqcZVGjTt37uDmzZtQq9UGy6nValRWVqKoqEin16KwsNBgmU8//RS+vr7w8/Ortd19+/bFDz/8UGu+ezGwICIiWWmIl5A5OTnBycmp1nwBAQEoKipCRkaG+MW/f/9+aLVa+Pv7Gyzj5+eHZs2aISUlBWFhYQDuruzIy8tDQECATt7S0lJs3769zsM2WVlZcHV1rVPeGgwsiIhIXhrxclNvb2+EhIRg6tSpiIuLQ1VVFaKjozF+/HhxRcjly5cRGBiIzz77DH379oW9vT0iIyMRExMDR0dHqFQqTJ8+HQEBAejXr59O/du2bcOdO3fw8ssv61175cqV6NixI7p164by8nJ8+umn2L9/P7777juz7sHsyZuHDh3C888/Dzc3NygUCuzatUvnvLENOJYsWWK0zgULFujl9/LyMrdpRERED72tW7fCy8sLgYGBGDFiBAYMGIB169aJ56uqqpCTk4Pbt2+Lx1asWIHnnnsOYWFhGDhwINRqNXbu3KlX9/r16zF69GiDy0krKyvxxhtvoEePHhg0aBBOnjyJffv2ITAw0Kz2m91jUVZWBh8fH0yZMgWjR4/WO19QUKDz+b///S8iIyPF7hljunXrhn379v1/w5qyM4WIiCSgFQCFBb0OWmk3yHJ0dDS5GZaHhweE+3pN7OzssGbNGqxZs8Zk3WlpaUbPvf3223j77bfNa6wBZn97Dx8+HMOHDzd6/v6JIl999RWGDBkCT09P0w1p2tToxBQiIiKracRDIY8CSfexKCwsxJ49exAZGVlr3rNnz8LNzQ2enp6IiIhAXl6e0bwVFRV6G5MQERFRw5M0sNi8eTNat25tcMjkXv7+/ti0aROSkpKwdu1a5Obm4plnnsGtW7cM5o+NjdXZlMTd3V2K5hMR0SPJ0s2x2GNhiqSBxYYNGxAREQE7OzuT+YYPH44xY8agZ8+eCA4Oxt69e1FUVITt27cbzD937lwUFxeLKT8/X4rmExHRo8hKO2+SYZLNkPz++++Rk5ODbdu2mV3WwcEBTz75JM6dO2fwvFKpNLnDGRERETUMyXos1q9fDz8/P/j4+JhdtrS0FOfPnzd7Uw4iIqJaaQXLExlldmBRWlqKrKwsZGVlAQByc3ORlZWlM9mypKQEiYmJeOWVVwzWERgYiNWrV4uf33zzTRw8eBAXLlxAWloaXnzxRTRp0gTh4eHmNo+IiMg0QWt5IqPMHgo5fvw4hgwZIn6OiYkBAEycOBGbNm0CACQkJEAQBKOBwfnz53H9+nXx86VLlxAeHo4bN27AyckJAwYMwJEjR+q0/SkRERE1HmYHFoMHD9bbmON+UVFRiIqKMnr+woULOp8TEhLMbQYREVH9cB8LSXF7SyIikhethUtGOcfCJAYWREQkL+yxkJSk+1gQERGRvLDHgoiI5EWAhT0WVmvJI4mBBRERyQuHQiTFoRAiIiKyGvZYEBGRvGi1ACzY5ErLDbJMYWBBRETywqEQSXEohIiIiKyGPRZERCQv7LGQFAMLIiKSF+68KSkOhRAREZHVsMeCiIhkRRC0ECx49bklZeWAgQUREcmLIFg2nME5FiYxsCAiInkRLJxjwcDCJM6xICIiIqthjwUREcmLVgsoLJgnwTkWJjGwICIieeFQiKQ4FEJERERWwx4LIiKSFUGrhWDBUAiXm5rGwIKIiOSFQyGS4lAIERERWQ17LIiISF60AqBgj4VUGFgQEZG8CAIAS5abMrAwhUMhREREZDXssSAiIlkRtAIEC4ZCBPZYmMQeCyIikhdBa3mS0M2bNxEREQGVSgUHBwdERkaitLTUZJl169Zh8ODBUKlUUCgUKCoqqle9P/30E5555hnY2dnB3d0d//jHP8xuPwMLIiKSFUErWJykFBERgdOnTyM5ORm7d+/GoUOHEBUVZbLM7du3ERISgnfeeafe9ZaUlGDYsGHo0KEDMjIysGTJEixYsADr1q0zq/0cCiEiImoksrOzkZSUhB9//BF9+vQBAKxatQojRozA0qVL4ebmZrDczJkzAQCpqan1rnfr1q2orKzEhg0bYGtri27duiErKwvLly+vNbC51yMRWNSMd2nLyxu4JUREVB81//v9IOYv3BEqLBrOuIMqAHf/wr+XUqmEUqm0qG3p6elwcHAQv/wBICgoCDY2Njh69ChefPFFyepNT0/HwIEDYWtrK+YJDg7G4sWL8fvvv6NNmzZ1utYjEVjcunULAHBpwYcN3BIiIrLErVu3YG9vL0ndtra2UKvV+EGz1+K6WrVqBXd3d51j8+fPx4IFCyyqV6PRwNnZWedY06ZN4ejoCI1GI2m9Go0GHTt21Mnj4uIinpNVYOHm5ob8/Hy0bt0aCoXCaL6SkhK4u7sjPz8fKpXqAbbQMmz3g/Wwtht4eNvOdj9YjbHdgiDg1q1bRrv6rcHOzg65ubmorKy0uC5BEPS+b0z1VsyZMweLFy82WWd2drbF7WoMHonAwsbGBo8//nid86tUqkbzy2QOtvvBeljbDTy8bWe7H6zG1m6peiruZWdnBzs7O8mvc7833ngDkyZNMpnH09MTarUaV69e1Tl+584d3Lx5E2q1ut7Xr0u9arUahYWFOnlqPptz7UcisCAiImrMnJyc4OTkVGu+gIAAFBUVISMjA35+fgCA/fv3Q6vVwt/fv97Xr0u9AQEB+Nvf/oaqqio0a9YMAJCcnIwuXbrUeRgE4HJTIiKiRsPb2xshISGYOnUqjh07hsOHDyM6Ohrjx48Xh4kuX74MLy8vHDt2TCyn0WiQlZWFc+fOAQB+/vlnZGVl4ebNm3Wu909/+hNsbW0RGRmJ06dPY9u2bfj4448RExNj3k0IMlJeXi7Mnz9fKC8vb+immIXtfrAe1nYLwsPbdrb7wXpY2y0XN27cEMLDw4VWrVoJKpVKmDx5snDr1i3xfG5urgBAOHDggHhs/vz5Ne+C10kbN26sc72CIAgnT54UBgwYICiVSuGxxx4TFi1aZHb7FYLAvUmJiIjIOjgUQkRERFbDwIKIiIishoEFERERWQ0DCyIiIrIaBhZERERkNY9cYLFmzRp4eHjAzs4O/v7+Out8DUlMTISXlxfs7OzQo0cP7N1r+R7y5oiNjcVTTz2F1q1bw9nZGaGhocjJyTFZZtOmTVAoFDrpQe8kt2DBAr02eHl5mSzT0M8aADw8PPTarVAoMG3aNIP5G/JZHzp0CM8//zzc3NygUCiwa9cunfOCIGDevHlwdXVF8+bNERQUhLNnz9Zar7m/I9Zsd1VVFWbPno0ePXqgZcuWcHNzw4QJE3DlyhWTddbn582a7QaASZMm6bUhJCSk1nob8nkDMPjzrlAosGTJEqN1PojnTY+uRyqw2LZtG2JiYjB//nxkZmbCx8cHwcHBetuY1khLS0N4eDgiIyNx4sQJhIaGIjQ0FKdOnXpgbT548CCmTZuGI0eOIDk5GVVVVRg2bBjKyspMllOpVCgoKBDTxYsXH1CL/1+3bt102vDDDz8YzdsYnjUA/PjjjzptTk5OBgCMGTPGaJmGetZlZWXw8fHBmjVrDJ7/xz/+gX/+85+Ii4vD0aNH0bJlSwQHB6PcxFt+zf0dsXa7b9++jczMTLz33nvIzMzEzp07kZOTgxdeeKHWes35ebN2u2uEhITotOHzzz83WWdDP28AOu0tKCjAhg0boFAoEBYWZrJeqZ83PcLM3vmiEevbt68wbdo08XN1dbXg5uYmxMbGGsw/duxYYeTIkTrH/P39hVdffVXSdppy9epVAYBw8OBBo3k2btwo2NvbP7hGGTB//nzBx8enzvkb47MWBEGYMWOG0KlTJ0Gr1Ro83xietSAIAgDhyy+/FD9rtVpBrVYLS5YsEY8VFRUJSqVS+Pzzz43WY+7viLXbbcixY8cEAMLFixeN5jH3581Shto9ceJEYdSoUWbV0xif96hRo4ShQ4eazPOgnzc9Wh6ZHovKykpkZGQgKChIPGZjY4OgoCCkp6cbLJOenq6TH7j77nlj+R+E4uJiAICjo6PJfKWlpejQoQPc3d0xatQonD59+kE0T8fZs2fh5uYGT09PREREIC8vz2jexvisKysrsWXLFkyZMsXkW3Ebw7O+X25uLjQajc4ztbe3h7+/v9FnWp/fkQehuLgYCoUCDg4OJvOZ8/MmldTUVDg7O6NLly547bXXcOPGDaN5G+PzLiwsxJ49exAZGVlr3sbwvOnh9MgEFtevX0d1dbX47vgaLi4uRt9hr9FozMovNa1Wi5kzZ+Lpp59G9+7djebr0qULNmzYgK+++gpbtmyBVqtF//79cenSpQfWVn9/f2zatAlJSUlYu3YtcnNz8cwzz+DWrVsG8ze2Zw0Au3btQlFRkck3DjaGZ21IzXMz55nW53dEauXl5Zg9ezbCw8NNvmXT3J83KYSEhOCzzz5DSkoKFi9ejIMHD2L48OGorq42mL8xPu/NmzejdevWGD16tMl8jeF508OLbzdtRKZNm4ZTp07VOpYZEBCAgIAA8XP//v3h7e2NTz75BAsXLpS6mQCA4cOHi//u2bMn/P390aFDB2zfvr1Ofw01BuvXr8fw4cPFF/AY0hie9aOqqqoKY8eOhSAIWLt2rcm8jeHnbfz48eK/e/TogZ49e6JTp05ITU1FYGDgA2mDpTZs2ICIiIhaJyA3hudND69HpseiXbt2aNKkicF3yRt7j7yxd89b8s77+oqOjsbu3btx4MABPP7442aVbdasGXr16iW+1a4hODg44MknnzTahsb0rAHg4sWL2LdvH1555RWzyjWGZw1AfG7mPNP6/I5IpSaouHjxIpKTk032VhhS28/bg+Dp6Yl27doZbUNjet4A8P333yMnJ8fsn3mgcTxveng8MoGFra0t/Pz8kJKSIh7TarVISUnR+YvzXgEBATr5gbvvnjeWXwqCICA6Ohpffvkl9u/fj44dO5pdR3V1NX7++We4urpK0MK6KS0txfnz5422oTE863tt3LgRzs7OGDlypFnlGsOzBoCOHTtCrVbrPNOSkhIcPXrU6DOtz++IFGqCirNnz2Lfvn1o27at2XXU9vP2IFy6dAk3btww2obG8rxrrF+/Hn5+fvDx8TG7bGN43vQQaejZo9aUkJAgKJVKYdOmTcIvv/wiREVFCQ4ODoJGoxEEQRD+/Oc/C3PmzBHzHz58WGjatKmwdOlSITs7W5g/f77QrFkz4eeff35gbX7ttdcEe3t7ITU1VSgoKBDT7du3xTz3t/v9998Xvv32W+H8+fNCRkaGMH78eMHOzk44ffr0A2v3G2+8IaSmpgq5ubnC4cOHhaCgIKFdu3bC1atXDba5MTzrGtXV1UL79u2F2bNn651rTM/61q1bwokTJ4QTJ04IAITly5cLJ06cEFdPLFq0SHBwcBC++uor4aeffhJGjRoldOzYUfjjjz/EOoYOHSqsWrVK/Fzb74jU7a6srBReeOEF4fHHHxeysrJ0fuYrKiqMtru2nzep233r1i3hzTffFNLT04Xc3Fxh3759Qu/evYXOnTvrvHq8sT3vGsXFxUKLFi2EtWvXGqyjIZ43PboeqcBCEARh1apVQvv27QVbW1uhb9++wpEjR8RzgwYNEiZOnKiTf/v27cKTTz4p2NraCt26dRP27NnzQNsLwGDauHGj0XbPnDlTvEcXFxdhxIgRQmZm5gNt97hx4wRXV1fB1tZWeOyxx4Rx48YJ586dM9pmQWj4Z13j22+/FQAIOTk5euca07M+cOCAwZ+NmvZptVrhvffeE1xcXASlUikEBgbq3VOHDh2E+fPn6xwz9Tsidbtzc3ON/swfOHDAaLtr+3mTut23b98Whg0bJjg5OQnNmjUTOnToIEydOlUvQGhsz7vGJ598IjRv3lwoKioyWEdDPG96dCkEQRAk7RIhIiIi2Xhk5lgQERFRw2NgQURERFbDwIKIiIishoEFERERWQ0DCyIiIrIaBhZERERkNQwsiIiIyGoYWBAREZHVMLAgIiIiq2FgQURERFbDwIKIiIis5v8ApIdOua5B/+4AAAAASUVORK5CYII=\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAI1CAYAAAA3qJjPAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsvXmcHFW5Pv5WVe/LdM8+mWSyEghhEQwQIgiIwQjIGkW8KiGgXjCggFz9cb0IIhDErxcUWdwuIIJKRMCFPQJugcgqCIQA2ZOZzN7Te3fV+f0xTL/Pe2YmZEKWSeY8n08+qak6VXXqbNX1PO9iKaUUGRgYGBgYGIwJ2Lu6AgYGBgYGBgY7D+bFb2BgYGBgMIZgXvwGBgYGBgZjCObFb2BgYGBgMIZgXvwGBgYGBgZjCObFb2BgYGBgMIZgXvwGBgYGBgZjCObFb2BgYGBgMIZgXvwGBgYGBgZjCObFvwfh7LPPpsmTJ4t96XSavvCFL1BTUxNZlkUXXXTRLqnbzsTq1avJsiy64447dlkdTjjhBPriF79Y+fupp54iy7Loqaee2mV1QliWRVdeeeWursZ2xV133UUzZswgv99PyWSSiIiOOeYYOuaYY3ZpvXYXWJZFF1xwwa6uxiA88sgjFIvFqL29fVdXZY+BefHvQrzyyiv0yU9+kiZNmkShUIjGjx9Pxx13HN10003b7R7XXnst3XHHHXT++efTXXfdRZ///Off85yvf/3rZFkWffrTnx7y+D/+8Q+68sorqaenZ8j7PfDAA++z1luHe+65h2688cadcq+R4O9//zs99thj9I1vfGOn3fOWW27ZpT90djXeeOMNOvvss2natGn005/+lH7yk5/skPs89NBDI/7B5Hke3XrrrXTQQQdROBym2tpaOvbYY+nll18e9py7776bLMuiWCy2w+q1u+DjH/847bXXXrR48eJdXZU9B8pgl+Dvf/+7CgQCaq+99lLf+c531E9/+lP1rW99S33sYx9T06ZN26ZrLliwQE2aNEnsmz17tjriiCO2+hqe56kJEyaoyZMnq3A4rFKp1KAy3/ve9xQRqVWrVg06Fo1G1YIFC0ZY823DiSeeOOh5lep/hlwup8rl8k6ph45TTjlFfexjHxP7nnzySUVE6sknn9wh99xvv/3U0UcfvdXliUhdccUVO6QuuwK33nqrIiK1cuVKsb9QKKhCobDd7rNo0SI10mVzwYIFyufzqXPOOUf99Kc/VTfeeKNasGCBeuyxx4Ys39fXp5qbm1U0GlXRaHSH1UsHEalFixa9r2vsKNxyyy0qEokMuR4ZjBy+XfibY0zjmmuuoUQiQf/85z8rtOQANm/evN3us3nzZpo5c+ZWl3/qqado/fr19Oc//5nmzZtHv/vd72jBggXbrT47A5ZlUSgU2iX33rx5M/3pT3+i2267bZfcf09APp+nQCBAtr31hOTAnNHnUiAQ2CH321rce++9dOedd9Lvfvc7Ou2007bqnKuvvpri8Th95CMf2Wns2WjH/Pnz6cILL6QlS5bQOeecs6urs/tjV//yGKvYZ5991DHHHLPV5e+66y71wQ9+UIVCIVVdXa0+/elPq7Vr14oy+MU/8IWp/xvqKx1x7rnnqpkzZyqllDr++OPVcccdJ45fccUVw153qP349b9+/Xq1cOFC1dDQoAKBgJo5c6b6+c9/Lq4/UO/f/OY36uqrr1bjx49XwWBQHXvsseJr7uijjx50r4FnH6jL7bffLq69dOlSdeSRR6pIJKISiYQ6+eST1WuvvTbk861cuVItWLBAJRIJVVVVpc4++2yVyWS22HZKKfV///d/iojU6tWrh3wu/OL/y1/+oj75yU+qlpYWFQgE1IQJE9RFF12kstmsOHfTpk3q7LPPVuPHj1eBQEA1NTWpk08+udKXkyZNGtQW7/X1T9oX/+rVq9X555+v9t57bxUKhVRNTY365Cc/KcbL22+/rYhI/e///u+g6/39739XRKTuueeeyr6R9PevfvUr9c1vflM1Nzcry7JUd3f3FuuPGOr5B57t6KOPFm2xpfsVi0V15ZVXqr322ksFg0FVU1OjjjjiiMqX+YIFC4Yc41vC7Nmz1WGHHaaUUsp1XZVOp7dY/s0331SBQED96U9/UgsWLNiqL/73qlc6nVaXXHKJmjBhggoEAmrvvfdW3/ve95TneeI6NMQX/3e+8x1lWZb64Q9/WNn30EMPVeZRLBZTJ5xwgnr11VcH1Skajar169erU045RUWjUVVXV6e+9rWvDWLifvWrX6kPfvCDKhaLqXg8rvbff3914403DnrOgw8+WJ188snv2R4G7w3zxb+LMGnSJFq2bBm9+uqrtP/++2+x7DXXXEOXX345nXHGGfSFL3yB2tvb6aabbqKjjjqKXnzxxUFfOURE++67L91111108cUX04QJE+hrX/saERHV19cPe59CoUD33XdfpexnPvMZWrhwIbW2tlJTUxMREZ1++un05ptv0q9+9Su64YYbqK6urnLdu+66i77whS/QYYcdRl/60peIiGjatGlERNTW1kaHH354xYCovr6eHn74YTr33HMplUoNMjq87rrryLZtuvTSS6m3t5euv/56+uxnP0vPPvssERF985vfpN7eXlq/fj3dcMMNRERb1EOfeOIJOv7442nq1Kl05ZVXUi6Xo5tuuomOOOIIeuGFFwYZRZ5xxhk0ZcoUWrx4Mb3wwgv0s5/9jBoaGui73/3usPcg6rd/qK2tpUmTJm2xHBHRkiVLKJvN0vnnn0+1tbW0fPlyuummm2j9+vW0ZMmSSrn58+fTv//9b7rwwgtp8uTJtHnzZnr88cdp7dq1NHnyZLrxxhvpwgsvpFgsRt/85jeJiKixsfE974/45z//Sf/4xz/ozDPPpAkTJtDq1avp1ltvpWOOOYZee+01ikQiNHXqVDriiCPo7rvvposvvlicf/fdd1M8HqdTTjmFiEbe39/5zncoEAjQpZdeSoVCYau+1Adw44030i9+8Qu6//776dZbb6VYLEYHHnjgFs8Z6n5XXnklLV68uDKGU6kUPffcc/TCCy/QcccdR//5n/9JGzdupMcff5zuuuuu96xXKpWi5cuX05e//GX67//+b7rpppsonU7TlClT6LrrrqMzzjhj0DkXXXQRfeQjH6ETTjiB7r333q16/i3VSylFJ598Mj355JN07rnn0kEHHUSPPvoo/dd//Rdt2LChMneGwv/8z//QtddeSz/+8Y8rhqp33XUXLViwgObNm0ff/e53KZvN0q233kpHHnkkvfjii2Ieua5L8+bNo9mzZ9P/+3//j5544gn6/ve/T9OmTaPzzz+fiIgef/xx+sxnPkMf/ehHK3Pr9ddfp7///e/01a9+VdRn1qxZhgHZXtjVvzzGKh577DHlOI5yHEfNmTNHff3rX1ePPvqoKhaLotzq1auV4zjqmmuuEftfeeUV5fP5xP6hNP5JkyapE088cavq9Nvf/lbopKlUSoVCIXXDDTeIctui8Z977rlq3LhxqqOjQ+w/88wzVSKRqHzlDnyR7bvvvkKb/cEPfqCISL3yyiuVfcNp/EN98R900EGqoaFBdXZ2Vva9/PLLyrZtddZZZ1X2DXzxn3POOeKap512mqqtrR10Lx1HHnmkmjVr1qD9Q33x61/2Sim1ePFiZVmWWrNmjVJKqe7ubkVE6nvf+94W7/t+Nf6h6rJs2TJFROoXv/hFZd+Pf/xjRUTq9ddfr+wrFouqrq5O9PtI+3vq1KlD1mFrMdBv7e3tYv9wX/xD3e8DH/jAe86VkWjpL7zwgiIiVVtbqxobG9Utt9yi7r77bnXYYYcpy7LUww8/LMr/8Y9/VD6fT/373/9WSqmt/uLfUr0eeOABRUTq6quvFvs/+clPKsuy1FtvvVXZR/DF/7WvfU3Ztq3uuOOOyvG+vj6VTCbVF7/4RXGt1tZWlUgkxP4BFuKqq64SZQ8++GAxP7761a+qqqqqrbLHufbaaxURqba2tvcsa7BlGKv+XYTjjjuOli1bRieffDK9/PLLdP3119O8efNo/Pjx9Pvf/75S7ne/+x15nkdnnHEGdXR0VP41NTXR9OnT6cknn9xudbr77rvpkEMOob322ouIiOLxOJ144ol09913v6/rKqXovvvuo5NOOomUUuI55s2bR729vfTCCy+IcxYuXCi++j784Q8TEdE777wz4vtv2rSJXnrpJTr77LOppqamsv/AAw+k4447jh566KFB55x33nni7w9/+MPU2dlJqVRqi/fq7Oyk6urqrapXOByubGcyGero6KAPfehDpJSiF198sVImEAjQU089Rd3d3Vt13W0B1qVUKlFnZyfttddelEwmRd+cccYZFAqFxJh49NFHqaOjgz73uc8R0bb194IFC0QddjSGul8ymaR///vftHLlyu1yj3Q6TUT9Y+LBBx+k888/n/7jP/6Dli5dSrW1tXT11VdXyhaLRbr44ovpvPPOG5FNznvhoYceIsdx6Ctf+YrY/7WvfY2UUvTwww+L/UopuuCCC+gHP/gB/fKXvxT2PY8//jj19PTQZz7zGdGnjuPQ7Nmzh1yLhppHOIeTySRlMhl6/PHH3/NZBuZVR0fHez+4wRZhXvy7EIceeij97ne/o+7ublq+fDlddtll1NfXR5/85CfptddeIyKilStXklKKpk+fTvX19eLf66+/PmJDwPb2dmptba38G1icenp66KGHHqKjjz6a3nrrrcq/I444gp577jl68803t/k529vbqaenh37yk58MeoaFCxcS0WCDxokTJ4q/Byb9trz81qxZQ0RE++yzz6Bj++67L3V0dFAmk9lu91dKbVW91q5dW/kxEovFqL6+no4++mgiIurt7SUiomAwSN/97nfp4YcfpsbGRjrqqKPo+uuvp9bW1q26x9Yil8vRt771LWppaaFgMEh1dXVUX19PPT09lboQ9S/UJ510Et1zzz2VfXfffTeNHz+ejj32WCLatv6eMmXKdn2e98JQ97vqqquop6eH9t57bzrggAPov/7rv+hf//rXNt9j4IfFlClTaPbs2ZX9sViMTjrpJFq+fDmVy2UiIrrhhhuoo6ODvv3tb2/z/YbCmjVrqLm5meLxuNi/7777Vo4jfvGLX9DNN99MN910E33mM58RxwZ+EB177LGD+vWxxx4b1KehUGiQtFhdXS3m0Je//GXae++96fjjj6cJEybQOeecQ4888siQzzIwryzL2trHNxgGRuMfBQgEAnTooYfSoYceSnvvvTctXLiQlixZQldccQV5nkeWZdHDDz9MjuMMOndr/XwHcOihh4rJfsUVV9CVV15JS5YsoUKhQN///vfp+9///qDz7r777m1elDzPIyKiz33uc8N6COia7FDPSrT1L9X3i229f21t7Vb9OHBdl4477jjq6uqib3zjGzRjxgyKRqO0YcMGOvvssyttRtSv+5500kn0wAMP0KOPPkqXX345LV68mP785z/TwQcfPLIHGwYXXngh3X777XTRRRfRnDlzKJFIkGVZdOaZZ4q6EBGdddZZtGTJEvrHP/5BBxxwAP3+97+nL3/5yxWr+G3p7535tT/c/Y466ih6++236cEHH6THHnuMfvazn9ENN9xAt912G33hC18Y8T2am5uJaGh7i4aGBiqVSpUfnFdffTV9+ctfplQqVWGV0uk0KaVo9erVFIlEqKGhYcR1GCmOOOIIeumll+hHP/oRnXHGGYIhG+jXu+66q2Lzg/D55OtkuDmEaGhooJdeeokeffRRevjhh+nhhx+m22+/nc466yy68847RdmBeTVgV2Sw7TAv/lGGQw45hIj66WmifuM4pRRNmTKF9t577/d9/bvvvptyuVzl76lTp1b277///nTFFVcMOufHP/4x3XPPPZUX/5Z+cQ91rL6+nuLxOLmuS3Pnzn2/j7DFew2FAUO7FStWDDr2xhtvUF1dHUWj0e1SpxkzZtB99933nuVeeeUVevPNN+nOO++ks846q7J/OMpz2rRp9LWvfY2+9rWv0cqVK+mggw6i73//+/TLX/6SiN7/V9Bvf/tbWrBggfjRl8/nhwzS9PGPf5zq6+vp7rvvptmzZ1M2mxWBoXZUf+8M1NTU0MKFC2nhwoWUTqfpqKOOoiuvvLLy4h9JOzc3N1NTUxNt2LBh0LGNGzdSKBSieDxOa9eupXQ6Tddffz1df/31g8pOmTKFTjnllC0atg1Xr0mTJtETTzxBfX194qv/jTfeqBxH7LXXXnT99dfTMcccQx//+Mdp6dKllfMGDHUbGhq2a78GAgE66aST6KSTTiLP8+jLX/4y/fjHP6bLL7+8IjsSEa1atarCRBm8PxiqfxfhySefHPLrcUBvHqClTz/9dHIch7797W8PKq+Uos7OzhHd94gjjqC5c+dW/k2dOpXWrVtHf/nLX+iMM86gT37yk4P+LVy4kN56662KRf3AS3Kol0I0Gh2033Ecmj9/Pt1333306quvDjpnW0NxRqNRQUMPh3HjxtFBBx1Ed955p6jbq6++So899hidcMIJ23T/oTBnzhzq7u5+T1uEga8h7FOlFP3gBz8Q5bLZLOXzebFv2rRpFI/HqVAoVPYN1e4jgeM4g8bXTTfdRK7rDirr8/noM5/5DN177710xx130AEHHCC+4HdUf+9o6HMpFovRXnvtNaidiYYe+0Ph05/+NK1bt078oOvo6KAHH3yQjj32WLJtmxoaGuj+++8f9O8jH/kIhUIhuv/+++myyy7b4n2Gq9cJJ5xAruvSj370I7H/hhtuIMuy6Pjjjx90rQMPPJAeeughev311+mkk06qfCjMmzePqqqq6Nprr6VSqTTovG3pV73NbduujCVsdyKi559/nubMmTPiexgMhvni30W48MILKZvN0mmnnUYzZsygYrFI//jHP+g3v/kNTZ48uaKFTps2ja6++mq67LLLaPXq1XTqqadSPB6nVatW0f33309f+tKX6NJLL31fdbnnnnsqbj9D4YQTTiCfz1f5wps1axYR9bvUnXnmmeT3++mkk06iaDRKs2bNoieeeIL+93//l5qbmyv65nXXXUdPPvkkzZ49m774xS/SzJkzqauri1544QV64oknqKura8T1njVrFv3mN7+hSy65hA499NCKdjoUvve979Hxxx9Pc+bMoXPPPbfizpdIJLZrqNMTTzyRfD4fPfHEExWXxqEwY8YMmjZtGl166aW0YcMGqqqqovvuu2+QTPDmm2/SRz/6UTrjjDNo5syZ5PP56P7776e2tjY688wzK+VmzZpFt956K1199dW01157UUNDQ0Vz3xp84hOfoLvuuosSiQTNnDmTli1bRk888QTV1tYOWf6ss86iH/7wh/Tkk08O6eK4vfp7wD1s9erVW/0s24qZM2fSMcccQ7NmzaKamhp67rnn6Le//a2IXz8w9r/yla/QvHnzyHEc0Q86LrvsMrr33ntp/vz5dMkll1AikaDbbruNSqUSXXvttUREFIlE6NRTTx107gMPPEDLly8f8piO4ep10kkn0Uc+8hH65je/SatXr6YPfOAD9Nhjj9GDDz5IF110UeUrXsfhhx9ODz74IJ1wwgn0yU9+kh544AGqqqqiW2+9lT7/+c/TBz/4QTrzzDOpvr6e1q5dS3/605/oiCOOGPQD473whS98gbq6uujYY4+lCRMm0Jo1a+imm26igw46qGKHQNRvE/Kvf/2LFi1aNKLrGwyDnepDYFDBww8/rM455xw1Y8YMFYvFKuF7L7zwwiHdVe677z515JFHVsJ4zpgxQy1atEitWLGiUmZb3fkOOOAANXHixC2WOeaYY1RDQ4MqlUpKqf7AHuPHj1e2bQvXvjfeeEMdddRRKhwODwrg09bWphYtWqRaWlqU3+9XTU1N6qMf/aj6yU9+Uikz4G61ZMkScf+hXPTS6bT6j//4D5VMJrcqgM8TTzyhjjjiCBUOh1VVVZU66aSThg3go7uF3X777VsVAEkppU4++WT10Y9+VOwbyp3vtddeU3PnzlWxWEzV1dWpL37xi+rll18Wde/o6FCLFi1SM2bMUNFoVCUSCTV79mx17733iuu3traqE088UcXj8W0K4NPd3a0WLlyo6urqVCwWU/PmzVNvvPGGmjRp0rAhmPfbbz9l27Zav379kMffT38PoK6uTh1++OFbfBalRu7ON9T9rr76anXYYYepZDKpwuGwmjFjhrrmmmuEi225XFYXXnihqq+vV5ZlbZVr39tvv61OO+00VVVVpcLhsDr22GPV8uXL3/O8kbjzbalefX196uKLL1bNzc3K7/er6dOnb3UAnwcffFD5fD716U9/Wrmuq5Tqb8N58+apRCKhQqGQmjZtmjr77LPVc8899551H+inAfz2t79VH/vYxypBniZOnKj+8z//U23atEmcd+utt5qQvdsRllI7yVrKwGCM4K9//Ssdc8wx9MYbb9D06dN3dXV2GA4++GCqqamhpUuX7pDrv/baa7TffvvRH//4RzrxxBN3yD0Mdg8cfPDBdMwxx2wx4JDB1sNo/AYG2xkf/vCH6WMf+9iQhlp7Cp577jl66aWXhGHi9saTTz5Jc+bMMS/9MY5HHnmEVq5c+Z52DgZbD/PFb2BgsNV49dVX6fnnn6fvf//71NHRQe+8884uS4hkYGCwbTBf/AYGBluN3/72t7Rw4UIqlUr0q1/9yrz0DQx2Q5gvfgMDAwMDgzEE88VvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMI5sVvYGBgYGAwhmBe/HsQrrzySrIsizo6OnZ1VSo4++yzafLkybu6GgYGeyTMnDfYFpgXv8Goweuvv04f//jHKRaLUU1NDX3+85+n9vb2XV0tAwODHYDly5fTl7/8ZZo1axb5/X6yLGtXV2nMwLz4DUYF1q9fT0cddRS99dZbdO2119Kll15Kf/rTn+i4446jYrG4q6tnYGCwnfHQQw/Rz372M7Isi6ZOnbqrqzOmYF78BqMC1157LWUyGfrzn/9MX/nKV+i///u/6d5776WXX36Z7rjjjl1dPQMDg+2M888/n3p7e+m5556j4447bldXZ0zBvPj3QPT09NDZZ59NyWSSEokELVy4kLLZ7KByv/zlL2nWrFkUDoeppqaGzjzzTFq3bp0o89e//pU+9alP0cSJEykYDFJLSwtdfPHFlMvlBl3vgQceoP33359CoRDtv//+dP/99291ne+77z76xCc+QRMnTqzsmzt3Lu2999507733juDpDQzGHnbHOd/Y2EjhcHjkD2vwvuHb1RUw2P4444wzaMqUKbR48WJ64YUX6Gc/+xk1NDTQd7/73UqZa665hi6//HI644wz6Atf+AK1t7fTTTfdREcddRS9+OKLlEwmiYhoyZIllM1m6fzzz6fa2lpavnw53XTTTbR+/XpasmRJ5XqPPfYYzZ8/n2bOnEmLFy+mzs5OWrhwIU2YMOE967thwwbavHkzHXLIIYOOHXbYYfTQQw+9/0YxMNiDsbvNeYNdDGWwx+CKK65QRKTOOeccsf+0005TtbW1lb9Xr16tHMdR11xzjSj3yiuvKJ/PJ/Zns9lB91m8eLGyLEutWbOmsu+ggw5S48aNUz09PZV9jz32mCIiNWnSpC3W+5///KciIvWLX/xi0LH/+q//UkSk8vn8Fq9hYDAWsbvOeR2LFi1S5nW082Co/j0Q5513nvj7wx/+MHV2dlIqlSIiot/97nfkeR6dccYZ1NHRUfnX1NRE06dPpyeffLJyLlJxmUyGOjo66EMf+hAppejFF18kIqJNmzbRSy+9RAsWLKBEIlEpf9xxx9HMmTPfs74DFGIwGBx0LBQKiTIGBgaDsbvNeYNdC0P174FAnZyIqLq6moiIuru7qaqqilauXElKKZo+ffqQ5/v9/sr22rVr6Vvf+hb9/ve/p+7ublGut7eXiIjWrFlDRDTk9fbZZx964YUXtljfgYWmUCgMOpbP50UZAwODwdjd5rzBroV58e+BcBxnyP1KKSIi8jyPLMuihx9+eMiysViMiIhc16XjjjuOurq66Bvf+AbNmDGDotEobdiwgc4++2zyPG+71HfcuHFE1P8VoWPTpk1UU1MzJBtgYGDQj91tzhvsWpgX/xjEtGnTSClFU6ZMob333nvYcq+88gq9+eabdOedd9JZZ51V2f/444+LcpMmTSIiopUrVw66xooVK96zPuPHj6f6+np67rnnBh1bvnw5HXTQQe95DQMDg+Ex2ua8wa6F0fjHIE4//XRyHIe+/e1vV74IBqCUos7OTiLirwgso5SiH/zgB+KccePG0UEHHUR33nlnhQok6l8sXnvtta2q0/z58+mPf/yjcC1aunQpvfnmm/SpT31qZA9oYGAgMBrnvMGug/niH4OYNm0aXX311XTZZZfR6tWr6dRTT6V4PE6rVq2i+++/n770pS/RpZdeSjNmzKBp06bRpZdeShs2bKCqqiq67777Bul+RESLFy+mE088kY488kg655xzqKuri2666Sbab7/9KJ1Ov2ed/vu//5uWLFlCH/nIR+irX/0qpdNp+t73vkcHHHAALVy4cEc0g4HBmMFonPNr1qyhu+66i4iowvZdffXVRNTPKHz+85/fji1gILDzHQkMdhQGXHva29vF/ttvv10RkVq1apXYf99996kjjzxSRaNRFY1G1YwZM9SiRYvUihUrKmVee+01NXfuXBWLxVRdXZ364he/qF5++WVFROr2228fdL19991XBYNBNXPmTPW73/1OLViwYKtde1599VX1sY99TEUiEZVMJtVnP/tZ1draui1NYWAwJrA7z/knn3xSEdGQ/44++uhtbBGDrYGllMb7GBgYGBgYGOyxMBq/gYGBgYHBGIJ58RsYGBgYGIwhmBe/gYGBgYHBGIJ58RsYGBgYGIwh7LAX/80330yTJ0+mUChEs2fPpuXLl++oWxkYGIwCmDlvYLB7YIdY9f/mN7+hs846i2677TaaPXs23XjjjbRkyRJasWIFNTQ0bPFcz/No48aNFI/HybKs7V01A4MxAaUU9fX1UXNzM9n2jif23s+cJzLz3sBge2Cr5/2O8BE87LDD1KJFiyp/u66rmpub1eLFi9/z3HXr1g3r22n+mX/m38j+rVu3bkdM8UF4P3NeKTPvzT/zb3v+e695v90j9xWLRXr++efpsssuq+yzbZvmzp1Ly5YtG1S+UCiIrGzqXQJi5mcvJycQouo3ZDrWVadBspZkie/RHqhsN/1DJpLwp93KdqA9W9nu3S8hyrUdxeUuPPyJyvZ+wQ2V7SuuPkecU4zx10n3gXx+eB03rTooJc9ZH6tsH37YG5XtFx+W6SwPPp5DX77zo324nh/iMsqvxDm+FCfgcBuKlW1/uCjK1d4f4eudwMdCr3MWvKZnZdv7u7jt3Ci396rTYqJcfDVvx9Zzm4Q681xvR/4aTbeEuD5HlyvbH9x7jSi37v/2qmx3Q3M5We6HlqUyalh6Aj/T5sO5XPUrvJ3aS5xCk+/v4/ocXlXZzkyUY2vO4a9Xtp9dO6myXS5y/zc8HhDn9E7lZ8+P42dNvCGnYwGG53+c/ufK9m9+fSzXZ7rs198efVv//rRHHz18M8XjcdrRGOmcJxp+3k/96rfIDoZI7d8nytfcx+O19Xhus+k/5DGZ2qdKnBN/6JXK9tqfTqtsh/4q28SX4znk8jCkYhWPj/IHZH3qoD6BXq6Pk+Ptvskyo2TPDL5ewz95Xmw8Ws6FKQ/APPHxOV0zuHKpw/LinHG/5+x6G4/mc6yyZE+CLTw33De5HZIruA06PihOoS8ew2PvD1d/hK+tkcU903j85ht4ntjjeB2xVkXEOZPv66lsl7+bqWy3PT5BlCvDaW6E7zvx4eHTdXd8gE8KdXF9km9wX6YnybWrbyKvn+kPcBvv/T8bRbl1n5ta2W75Ga/Tb17B6/T0n8t1P9fC93LyXJ/gi2+Lciuu5FwKdc9zfToP4nOq3pKJluxSf5u4xTy99svvvOe83+4v/o6ODnJdlxobG8X+xsZGeuONNwaVX7x4MX37298etN8JhMgJhMjnk4PLDsOLP8wPb4d4cfX55eLs8/Ek8zm87fhDopwdhhd3jJsmGrKHPccJ8MTC850gvPgjcnG2Q3wNP7xAnaC8Nh7zwX1tWE/0F79d5DZRYah3RC4u4npwDOugtz22neXjfsDnISJy4D3n80Pbw2jTX/yOeD5ePLEN+q8N5eC2jsv94POV8RT5rCEuJ/pOPgL5HO4zbBM7JMdWIMb1syNQzuGH9fm1Zwjys+OzOgGfVo63QzE/7Me2ku0Yi8u/dwZtPtI5TzT8vLeDIXKCIVKRktjvG2Z84Jj0aXPTZ0GbQd/gGCIiclwY59BVTpDbbkv1wfHm4LZ2Hxx7OC/skDY3cZ74cLzinBWnkA/S6tphePGXZP878BwqhG3CbaDPhVAMxzIf1F/8uObhPLHhRW1pa4UPB3kU2k5bCxUUUyG+nr5Gyfrg83F9cG7rY8YJwjsF1lmfrc9h6H8L1oAw7HdkmnG8l1OG+ljy2ngNJ4D18YbcT0RkW7Id3mve7/JY/Zdddhldcskllb9TqRS1tLRQIK3I8Svy9chftpENPOKzMF/calgMMrIRgm3wBdjRXdmsWiUbPL2Sf5H9KHlMZfu0vV6ubFtaVkoL3jGhjdycAfhA6GmXszTUwxVfvo6/EuObZL3f7GZtNLGGv2xCbfxrLl8vK5R4i7czOZ4t+UY5UCIbuF2Db0Yr2zUr4KW7WX7lUA//grX9tZXtQK8cZPF1fI3oKk7gQZu7eLtGsi1xWETSUJ+X4+NFuZb1PJnydTxBPF73yNfaI84JRWBRLHKfxzZxPXMNfnGOnefFAfs81C4X6X93NlW2wyFeVFO93PaWK/vVD82ag6Zz8rJcqIO338jwffwpLudvk/VeU+7Pw54tu0TUSqMVw837AYSD2o9ll1fhWJLnggd96wa0xc6FH6qwMLryQ5wIFsl8Pezm0ykZk1+WxSh+vcEPuRJvF5JyrJQSfEGsq9Kk2HIUXqBFHnxYH5WSa1egl8eer57HdalLvtgKBW4vL87Xzjbw+uBVybafEOgcst6kvVzwOSyPj/kDMM8Scr2yilzvQpmfu1itvciAuSjDWo/vO2XL+uC8xXJWge/p+eU5Nr6rsQpl+TFBw/zecPqgEbR+9WW5A8vw0Up++Rq24CNGf99UrqW94yrPVxxcdsjzt67Y1qOuro4cx6G2tjaxv62tjZqamgaVDwaDJte6gcFujJHOeSIz7w0MdiW2u7lvIBCgWbNm0dKlSyv7PM+jpUuX0pw5c7b37QwMDHYxzJw3MNi9sEOo/ksuuYQWLFhAhxxyCB122GF04403UiaTMelVDQz2UJg5b2Cw+2CHvPg//elPU3t7O33rW9+i1tZWOuigg+iRRx4ZZPyzJbgBiyhgkVsl6cBCLWgbqH94qNVI/UP5QIMD38ZiUl47V8/nNcbZwvSYOFtuP2l/SJzjgsFOOcbnl/NgOBaXhkHWBm72SJhFpUBas9ix+QHLcanpDQcXDJLKYARDIVeUUw6UA+ObEhiLKU17ssHqSKFhjy6tQpt4YTCwivHzeSGpTaM+Z0NzOY4UuTyoN2qeNna5pj3aJbgGbJbDqM3KZ1BgaIdW3+WIvHZDlO1HMiXuo74QC8nlkGxHheYWYIhaDstyaGFeFwBrbOzjqGyfZl+/TUXaN4w4uIOwPeY8IpuX4x1trz0P+q0IOr6uu1pcLtfN/ZHQdFA/6KX5OmjbMO/vy8m1QtQHdW+YF542Ze0c1LsMgzch9WMP1itfhvuxUAN2AVr/omZc7uC5pWvEOJ8U6OaBPjDAy0h7oOczU6Ac17uQkOXE80JfuC4/D+rXREReFfdL2NfD9czKcri2BlvBFgAcOQIpOYmLYNgeBnsZnNue9gZ0SnwfVeDns+LS+r8chXLY57DmqqBc47INQ6/h4aK++MB90ETD0xZarM+7zeg6wxYR2GHGfRdccAFdcMEFO+ryBgYGowxmzhsY7B4wsfoNDAwMDAzGEHa5O99w8Gc88hU9csM6TQqUlDc09eXLaq4Xwk8XXbv0QD98vWyBaZl3iuxWh7QvEZGV4L8D3eCny0oBeSX5+6oI58RsfB5Z7Z4s02BOnDkc4Q7WrMUsQIoMmaHC8LScGoYechPSFcgqMEeKVD1KBYMvAsdy7EJYHl8tipXBTx3pt3FJGQSjFGUrcQzqIah6jeovxeGC0BX43K5mYK78zpDldNer/RKbKttPrOfAG74gj8F8jWzHUBdQyAWgdbWxhRTyv3vHwX4uE+yWFdpY7neT7Hfnk0FHdieUS9q8BynIAQnMAqq/HNQknij47eG6oQ9X+NvHnoJifcl2SBmusY0HXL4aYn6keIz7srrfINwSJCvKaO5cHjxfiZ9PwSn+LnlOCanwRl58SmuiolwxC/Qz+tqXwF+8INvxwMi6yvY/QrO52uPk2BO0ObTpuGqewxtXSsocpZH9kzxe1wRbRDGc36Up3MbOUxAMq05S6Thvgz0wTpI8H3N1WtwLeHU4MYh50NUjyjl5DjCkcuzq6YB/rufXru3BuyKFMVFkXwbA3RvfCQpkQbGmEZFTUIPKbwnmi9/AwMDAwGAMwbz4DQwMDAwMxhBGLdVfjNnkBmyKrpcWj4Eu5mRzk/iY1cePkmuUjxXLalaTA/dIyHIehL+Nh9javgUiV2Ub5G8ltLAu1IKFNsTwt9IabQlhJifEeyrbK/auF+WaE0yRZf1MkRVquEwEaL3++rCZKz6PVdSsZCNgsQr0Vq6eny/xlvQEEDIJtGmgR167GMNY9EyRhl2m9z0t5GQxjpTd8NJBBiIQIj2fawIabFxSnFOK8rXLSXhY4O3zjZIjKyX44oUEPx/KNESSgkeEIIqf7jEgcgwkWT4pJiSdjBbLH2vgeOA/auBoj/q1651+HSjt7Fyr/u0Fp0jkWERuUQtjm+XnSaeZqlU206yRDm28glW/leNxg3HbiYhciN7mT2Pcft4frpWR+zwf0/ho6e5FIAS3DDoqaFgPQ/FmdC8UiLwHnkfhzVy37CT5rDZY6Ltlfm43pkXKw0iRkNcDI1+6SSmVloAzz1XztcMduvcU0ty8f0NHEipKwyJVBmkzJ9tkODkSvToCKVkfp8DrLq4BwQ6IcpiRMlwJ1208oHs4oSRQx1FMUe5wUjJkr93I9ypHYf2NSEmo0MgXj0HOFwe8LXCcEhG57w47JZttWJgvfgMDAwMDgzEE8+I3MDAwMDAYQxi1VL+y32ViPY32HYaqwiQOTkFSVWhBq3qZPg+31YhyvixbwG5oT1a2Xx7H1KpPywKJltjFBAbe4DLFGi3LHSRymBbjyBIry9NFualxPrYiC9bsUaCe+yRVFYBnxTqUZMZScvLcRuF2SC4EdJmd0h4WkvRYCY6OYWtOFH4IOoLJQ+w0c59+LRMdyi5odd2VkfR3AKqElswY8ENP1oH1QcoX6UN/Sp4T6GDzbgtSg+kW4WjVvznDckyqj89JlDRKFG6FWbV02h6p4c3A+wegroUaSeVmVD/nl1W7J9XvOUSWM8gxQwZhEgd4U0+G5Pb08DFIPOP5NHoX5pMrD1WQz0iLcX+GB72Cytowr3R6GiUxXw7kAc3624VsfYEeHhRucPggXjimykBx68lkFHgY4Vi2MahRWdZnWYpTGqOcktESf6G0gWO3DAF8dMbeTjMdvi6TrGyjdwURUQmcATxRP/R60DIRQn1QfrSK3Eeu5gmCFLqHAXNcKa3g2oMBfERCPu2zGu/lg7S8pPWrBTIXylBo1a+vcSOF+eI3MDAwMDAYQzAvfgMDAwMDgzEE8+I3MDAwMDAYQxi1Gn8w5ZHP7xFtQctAXd8LgGCle4OBnOKCxq/rJKjP1FZzUpQECE5OUYuuBqKVAxGvMJmQpbkmOeBat2wzJ8BAdx0iovY8C1tl0P3wPq4WFRAjy2HkMUz4Q0TkBbhgCYJ7BbuhTESGs3NSQyfwcTRTAH8atChw8cHIfVZp+Khm/l4+JxLUMqqANubkMeod6Gcp6UflhvkBUdd0Cnwtv+Zyafdw/8c2JCrb6BZERPT3tqmV7a5evg9q97pbF0aILBQhuYrWJDim2yHjiJ+rJlzOiIiyXn+f5TzNtW03QaCvfx5qQ0pEPYvGYByBPU+2QfZhJMF2EQp0ZleTyjFJjwPDTbSt5ieF2qtdhiRXEBVuIJoaVxZcxeB5fLo7H4xLdJvV1x5EsBsMbVLcDv6sHl0PosfBPBsczpDhB8F+0DMBUIfHNSoBa2m7FkG0XMNzZmZidWV7Vf1EWW/MdwN9gXYdw9qBkEy+g+8D7Pv+C8I5W3CJFdENQf8XSYg0mwNcm/3gAkpZOdrR3RufG+9padPbfdcN0dWNY4aB+eI3MDAwMDAYQzAvfgMDAwMDgzGEUUv1K9siZVvCPYaIBBVTbmRezsoO7Q5GRORBPnRfLbvwlUKSdkL6JAP5wNPg44NR3IgkA1ioGZq+K4+TdLWX4mtj5L7V5QZRri3L9C7e1QdUcXmSbB9fDrp0X6bYQrZsFBsSPZeqgFqCJvG0PPIi6Qm4sGTHy2v7IelI1RpoVIj8V45LGcEGKi5fzxRbbVj69XSAPOPBJVxg5aysjJjlBrkdvSjXpwiJjwpJ+QzlxmRlO1fDv48xGiIR0X417M5XhGTYSPW7fumSKFyn+rhNdFdRpJr3jfJ9lgU/APWR58x+V6vpK+6e7nzlCJEKEgUics4UktyGrgv9CwltQj2S/1R5LqdAbtNzsJfi3M4oe4nkTJqkJiLdQT8FerjeGAWTiKgchjEBUhvmdiciKsf4xgGQ3oSMV9IicUIkOBUBV8HC8PXGeeYHF2M7L88Jgp+pk+dr5+vk4MPkUYVqvvbUKFD95SQNh815mKeaHCOkXHimUhVEEy3LdkTXTBcSXllFdJGU9wl3QvRVdIsMyoK4Zlpx1jiKST7fC8v2wXdFrp6PhbQkPTY8H0ZpxTYoh4eWePUkYsPBfPEbGBgYGBiMIZgXv4GBgYGBwRjCqKX6AymXfH5XUMpE0oIcE/MooGAxAQYRkV2GyG1Aq+gRszC/ewnyU08Jbub7axbawlITrGTLENnJ7hyeEvvXpubKdiSiWWjnmV6q62VKvwyW5apXcmIikmA7P5BydGtcTPQMzwDn20UpI1g5pk69ENdNt0oOpJkOtAtAv5Z5W7e8zzXy9TDhCNLnRNI6F2lLfx9EuIrK8GseWmAjpQmR8vRnwCRCIsKfVu7NXpZn2lvZ+h8TMzVplsM2WP7aNdymblBKAi5Qwx1gMi3GoMbot79rLZ7WI17uJihVKXJDimriUuJRDrdNuQyeEA1suZ9pkGMlFuVzYvUcxrKYSIpy/j7eDsCQz0zgNhzf0kmIQjX3O0aGJIicqdPVSA/nIdmNCshOdHL8d6GO5bV8HYzxmJZIJ8qTobG5q7LdVpbRSW3MMd/Bc87JQyS6pAwh+f1xL1S2Dx9/WGU7kBLFKM+5aoRVf2eO+8FJa1b9ca73wYm1le3lzj6iHCYS8zfxBLDLfL4b1CMg8jauFbmpNVBGzufUJK7fB6e9XdnO5qT0JKJsgtyE64hydDoePQGgvcOa7NkAEU43ctsVwVPM1TyABjyCXD004jAwX/wGBgYGBgZjCObFb2BgYGBgMIZgXvwGBgYGBgZjCKNW4y/UOFT2O0QUE/szLaCHoRYFmqqeFQv1Wge06XJE/u7J1/G1D21ZX9leU6yDa8lrY8Yl1GQxO1VgUhpPocJ6fqYPNvN93srNEOUSMdAl4xCFDJ7PzkmNCl2IlF9L9wawwd0LXdTKoHkpv9aQmEXKB5HQNJcYtLHwwGXSbajm7ai0e7DdodvO1nwz0aYCdcThNDciokAPC4S+HD+DA5G+gj3yGfxtLGDGNvID9k6VbaLXr1LPONxTc49Cdx0XbDRsLUgh2oy80N3C+yGDWLlKurC9kJ9ARETZgktEbUPWbTTDyVjklC3q7JHzHh1di5Apz7eZ9exwkzYQS9wHuSwfi2dkMYxGZ0Nzepu5/dt7ZX3q4RxfBk6CcRzokWOjWMXXC3dCtDfN5c7fB+5m4FIrbDtychyincG6t1hs9+U0OybYdrDamKUuJV8Lj0DbYfRAPYqfDRFJ3QauT1WA7VjaAprNFtgAvdI3ng94cl3DaVbuZIEbXSSD3XIC+dN+2IZybWw/4h8n05ZiH72yke2vpjnrRDn0r1ap9JD7nT5px+TPcTvimkQd3aJcuY/7L9ALbtNFvKmszoCdii09mYfFiL/4//KXv9BJJ51Ezc3NZFkWPfDAA+K4Uoq+9a1v0bhx4ygcDtPcuXNp5cqVI72NgYHBKIGZ8wYGexZG/OLPZDL0gQ98gG6++eYhj19//fX0wx/+kG677TZ69tlnKRqN0rx58yifzw9Z3sDAYHTDzHkDgz0LI6b6jz/+eDr++OOHPKaUohtvvJH+53/+h0455RQiIvrFL35BjY2N9MADD9CZZ5651fdx/RZRwCJfTovcZ0NiFvC9KYsEEZoLWHpo/iPYI69tF5lCnBrpqGwfEGI6/m7N/WO4pBVOnstVx6RrUjtE5Duq+s3K9iu1M0W5D1S1V7ZfjjDtVGpiSisQkXR+uRMawg9uJkFJCZdiQNvDIxXjKF3I4eED6pTKw0eGQ7nAhkhfdpp9MUs10uUOoyaWEvzH+t6EKBcGeUa4T0bAzS8kZQQLXELLk/hllFvFdcjXaw8BUQbzELmvHJXFxkVYEthcw21fgKhfypa+N5jMxIpym9pl2d4K/pwaZ3ey1S2T4AKyPs3+ftow49t+kft21pwn6ne7sgJEDdV9Yr+yIPkNjOVyPc8l5cjGUNDv6BWMiV2IiNwwJE2CriqC+100LNcQ7FOMtIeR7SytC3xZmFvgeiYSuxBRCVzckOIOpEAO09z5LPDjUjC1Xc0dOlzHa5HXxjS3PwvSRb38wWaDzyhGjPN0SRWnHTz72i6W+ERym/4KVjZ7iuy6hm1FJCPi+fqw7bgd3OBW+rIBlPYGRJfuMrh0W0Gp8QZ6IBJgFSRTi4KEGtF0YUAxyTcOJ+LyIEqvMDbJRplFnlKRoHdF5L5Vq1ZRa2srzZ07t7IvkUjQ7NmzadmyZUOeUygUKJVKiX8GBga7B7ZlzhOZeW9gsCuxXV/8ra2tRETU2Ngo9jc2NlaO6Vi8eDElEonKv5aWliHLGRgYjD5sy5wnMvPewGBXYpdb9V922WV0ySWXVP5OpVLU0tJCvpwiX1kNikyGySnKm4GXAwt/tIolIrKAlnbXb6xs25NqRblQJ1/7V89zhKoXpvKiFG3VKHOIoof0PlqZb1xdh6dQEFi6lTleMKNt8mH/2co5qRMdfFJoNVovS0tmH7B0vs1MNZVrZZtgMpHIBua34uv5+YJrusQ5qpe/yqwIUK+alIvJUpw+4KSAdvSnpAVuKcZ9aYFFL0ZpIyIKt0P+9UaI2gVlvKAc1mXwLPDQghuapBSTbe9CEiE9qQuipwiUL9TV7wc6OixpS+wjGyIq6gl30Mp/bYbpUhGFUUu+tLLQREREuWKZiN4ZvuK7GMPNey+kiEKKyp78JkHlIhCE6HOO1mgAC3KTJxNsyu/6pPSC0S4xkwr2WzYvaduqNLqe8KYHdDNG2iMiyjXzmIhuBM+OhEbbY/dijnm4nM8v1yFl8yCtb2Er8fbN0mq9KsKDrz3BFDXOBdeVbV/rcNuVgAr3Apq0AlMV6fjwJB7IZU96R9hFfo5j696obL8WmyyvDbJlfGoPH/grS4EYoZNIyoeBPngHRFHG0+YmRIZtaQBrey3PvUiGAxFJg50gudiarAH9ihKxCmhjGCKX2jA0vAR3UlmL9jewdmiq0bDYrl/8TU39i05bm3QjamtrqxzTEQwGqaqqSvwzMDDYPbAtc57IzHsDg12J7frinzJlCjU1NdHSpUsr+1KpFD377LM0Z86c7XkrAwODUQAz5w0Mdj+MmOpPp9P01ltvVf5etWoVvfTSS1RTU0MTJ06kiy66iK6++mqaPn06TZkyhS6//HJqbm6mU089dUT3Ub5+itXJSYo61M40XXYqH7O6mYorJiQNZpUgOUwTU+sln/zdg5SuE2GO5YPVHLxhaXginjIoeM0AkDqzipJ/KQPT+HqKv4qKUY2qAs7PGibADea9JiKKbILgOWAdGtik0UkQvaOY4HK5Wm6TeERa3tt1kPCjpHlbAFD+cKsgmU8BuGvN2hitnG2w6G2p7hHlUkmmCtPTuA7R1RjoRNatUAu5r9fwtucDK9m8bHsPxgZSvpYcWhTx8TOVilwHt4/vEwlqVtvgGeDm0CJcy8sOSZuSAeYgw21oHS6n8ORAvydIRqOC3w921pwnIvL1WeQUBwfwqY5xf8TAwt4Ba3RlafRnDjw4Crw+hLS87Uj1l2JDB+QqpbV87FAfpJEDndxP0Q1SHlAW93VkMywQWgKtEngJhNsgCFE9163Uqz0r0O7ta1kWQmmUiGgzBIcJt/EzoAeDs1bO+0cPPKCynVgN9amTMhy2CVrLl8DjwNMC+GAim+d6J1e2gx3a2hzgv7NtLJ1G/Vwff16T6+AxLEhahcG5fFmZGAvrvbaV17sZmbdoOHg9vXyfLUw7F/oIpQIrJQO8hTaAvQw2lzu0lEzE7yG1lZ/yI37xP/fcc/SRj3yk8veATrdgwQK644476Otf/zplMhn60pe+RD09PXTkkUfSI488QqFQaLhLGhgYjGKYOW9gsGdhxC/+Y445RvjH6rAsi6666iq66qqr3lfFDAwMRgfMnDcw2LNgkvQYGBgYGBiMIexyd773gtJ0eHRpsbIYrQr0He3jBF1GPHBJs4sNshxIw6EQiyidIMr6NB22CLqWiD4HEqWtJeFAdx3Ubjtz8to26Fd4jgMuJ84WXMUCKYyypWlr4J6CUbLsMtoSaL6UoOurKm4Tv5b0RMAaxr9E22+XhtbbLT0JDgayguQ3pRhovXoEN3CrKVXxM/khnLxdlH2EdgJoU6FraHmXtXwf9Jerho6+RiT1OSfMg0ZEUySZECrsgCsP2AjYmqb4RqE/wmO/O9/wuuRohaX6/6HLHhFRADT2vhIvWyJKozbULD/YXICLmh71DBPUiL6BKHPRWqnDWordyDDZVznB9dRdxQq1PJCKcV67fGHpD2uBK2OhGpI4wfpkRXQbG2gHsF1RPjn4ggm+VynDi1SIA0NSqUYOqg9E1lS2fzcB2l4zG8K5UayFCIYQAU+P3GcX+TkmR7gSz+u2U5ikZx+IPvhvLlis0iJfQn1ytRApbwMky0nJ9ilUc/1CEbbfsaplBNFyFNbJMEtaaC817NpH0vVQxaSdQb6F7xsHmyRfF28H+rRkR++e4haHZ+YQ5ovfwMDAwMBgDMG8+A0MDAwMDMYQRi3VX4xZ5ASsQYli8pDnWQFNii4x6CJCRFRoYColtIl96UoaNZSZxNc7bdLrle0q4M+X18lrZ5uRVuP9GHVNjdPyMq/kOqxOscsIugUREYUhj3W6Gi/Om2Ut4lx6PNNqmNjCC8lySH8jpYy0theUXJ5d4oJulLdz9ZJeCkDyHHSlxMQ3nibhZBrhb9jsykkaLJTlCkbXcpsUkxCRT3MB9UH+dg/GTynKlJ/uHoM0HVKsmAxIx6RajvT1JlDTgT7NnTMErkm9QOXlRDFywe2ztwSREoGqLlVJWrbW6aeks872c+fbmXCDRBQi+tD4NWL/y9XsUuaDZ/N1M+1b2F9G5LNgvAVBOtApaqR70T0Xk9D0dEj3wqr80OOgFOUxqUs81MzrgOfj/iz3yQr50sMk+sEFpkeeE+gFaSrH5XwZOc8U5JvXkwgNwOmTbnovZifzfYBixtz1evV8vXyNluk9le3OkIzYSTbPzTiEAC3UapEJwTV5ziQeGxtze3EZjVoXrnWYyh7WHr8mr6I758Qans8qK+uNkoXXywmlMIqpnZWLij8Hsg1Im1ZaJnGz8ixBo/TqRrHttYRE71bH3YK8gDBf/AYGBgYGBmMI5sVvYGBgYGAwhjBqqX5/WpETUBTYKNN1hluZBikmIKEG5EH2ZaXFa+gdzmtfbmfLUX9vsygXXc208kON+1W2P7vPPyvb8Q3y2k6BKZdcI9cnsolpmT6lUZBwiVSeucVwUtI33SmOX97QybRR+8EQIcsvqapIG1BIYNaqU+tOHqKfFbgODliF6lb9FiSj8LVzv/jTMp+0U+TzLEwyAtcTyXuIyJ/htkcqtlCSQ9SBfOBotevkwEpWe9ZyBCyoN/DFkTIX1rhaXUV0Rr0YJnWBCqHHCUZDJCIqgLV39QSO+qVelUmjfMAAfriaLfRfie9T2Q50yTHzaPf+RERUTBeJ6AXa3eBPEzklor+vnSL213dxf2zsZbeG2gSMKS1ZFDncNiHwuChrbGiubujvnxJEALU0rxhfjo/1TWAKt/ZFpodTE6vFOV43DySk2a2SvH8pDlR0hu8T6uA6pGfKdShXBxbfTexmozO/+W6WGKwEjnFuKzep614AuB7S4kQkLe+h7RLgufTWWzJ/Q2OBafK/d02rbDtZTaKA9i+CBmNhFEaN/R4uqioip3leoMdMACQlNx4V5XIt3P72pPFQT7hWUlYgj+s7VDsalN48GKEvMw7awYX1V5MoBqI6ulv5Rjdf/AYGBgYGBmMI5sVvYGBgYGAwhjBqqX43aBEFLCo2y3SdmSlMQ0XqmAuN+TGZhaTYyKqvbAaBVsnVSYol18TU13c/8GBlu97HtPYD9R8R5yBtW4oz/ZICZmjcoRvFOZuWj6tsVwX4eRwtyUQSAnv0TUjyAWB5/NWS3yzG2fo4MxksfbWAH6V/+ocs50/xkCgnZax1H4ZttYcOnkMkk9og3eUHutaNaPQWAi7XFO8Th7qDyco2WvILCl6zVsYEHaUk0HcBpt6CPZpFMHgjCKvvwPBWs9kSt6mKQI7ulDwHraF7uri/kp5sxyLEDFnaMaOyjUGaClJFolNqX+yvS8ClXw9b01EM1f+vVNCCsUATOg7Iel1MaytHeoAQtGd7KzdmtdaFkc0QWAfo61A7yDV+vW+4XLAPvWfAk0ALDuPr5TkTaYPgMGXNQj+FawIm0+I5Y6Vl+4Q7+JzSBl58lCZRUBzWAQgsJiz8taTuedDeAr0QcCqilauhIdFd4H6xwtJa343zGtMU5L70grLefmi7519nGag5AYmP2qR8WIqBhwWMBavAddADYOE4W9PN75FmbY0Tidf6uN6Y2MnJyTVX2UNLqiojrfrJBg8lvAQoBXoAn4EgVK4WnGo4mC9+AwMDAwODMQTz4jcwMDAwMBhDMC9+AwMDAwODMYRRq/H7cop8ZUW+PhkxySqyflUq+obcrkpL4cYug4DVye42dkHaArgQBe+tQmNle0n7IZXtcJcUkPN14FqCQepAM97YJe0UnBIf29zBxxp9UjPLgqtfooPFnp59+FnVaulmgtHeHIjaZVXpkfvQJQaiUGHOk7KmaxXAJaqW74u6FtHgCHSVe/rBra5H6lqFar5eOQG2CVr4s1A3N3InJEexQXOzC9IdCW0nrDI8NzSJq6WOt9t7uD5hFi9zE+TYivlZVNvUx26Ndg/3UbBban3o/oV6HkZTJCLyg3nDEbXszrc6z25PSnNhilj99VHW7hm5z/MTWX6iqio5iGxIhlRG/R/aTE+g5aVZe21q5nmfDTWKcmXoe0zYg8mw7Jg2pqAOGDEOh6uugeM8KSb5fHRFJSKywR0WE4yFuvj8QrXseCfLhh8qAJH7+uS3nVsDkU8xWCYmydIi9+Wg7fO1Q693RDJCZRlcEuvDnOBoVWqcOMe/Zj0cY3fWQW0Czd88kV2yg128huvzJwE5qvxZ0PjBlTnYq61dUK6tk9ek5rdeE+UC3QdVtr0edskVbsVaBNn4erDdADsDsmQ5jLYY6IH1ARN6RbVoj++691kmSY+BgYGBgYGBDvPiNzAwMDAwGEMYtVS/XVJkkyJyh6cuAsGhKWEnrz0WXsMFGq2k+X1BRCgXeLC9ohz5b61/b3FKsYqv7QXADQNcYpJxSVt2R5hCSibRHUm6uNk2hveiIeFG5TOUIa+8By5ISmuTUCtS7exShq5r/o3dhFCQ9MTXw89kF6UbVRnkBjcAUchKw9PPsY18LNfA/bChUebBRlesYBdfG6Pc6eHKXEiKY9eAG5Wn8fsAVV015H47L6/dFGJXz1SMr9cdZArSC2hUHtQVx22oW/ZlDqjC19IcZVK4tmnR6l7MTSYiony+RERvDvkMoxmFWkV2SNFB9a1i/8oqbs9kDVOruUnJyrbuamknWHrp7uMxmtyszZkQn4dSWb5h+LUHx3WuhrfDm/jauQZZn/Ez+Zmyr3IEO3eC7MRCLc8zGyTMQpKvVxgnpYdcE489u5qp7HJU0vb7t2yqbL9SnFDZ9uX5Gdxaee3JIabWl1bDXNKkwAK41+IYXd3LUplwgyMi8vPzJYO8prRria1QgnQsXGcxwZImUUD7o0utikGitrC8D8ofLRP4ua24TNLkZ/WCLFhvfED1e34tImOM61cAqSe5RvrkysRbIK2ALOgN8+bWgq0OC/PFb2BgYGBgMIZgXvwGBgYGBgZjCKOW6ld2/z87L636nRz/Vsm0RvXTiIio0ZNW1GgZS82c5EdP5hPoYqr9sdZ9K9sRP9dBt6JOruDtYhVQz5BEoS8rKWWk4BtizBn1KOllkIbzgnG+sV1Gy2P52w29CZDiLkclzV6qQVNmoKdAbSg1y/r42sHMHOitckLnl8CbANhAjIZXbJKJfdLN/HzlSUz5HdwgKd/VMZZaCrUQCRAoO0+LCojWvh60XSnO28WkfAa3itunkOT9vrRs7zrg/B5q5cROaIGbq9USwOMlOthzQ89vXgYFJVOGXN5oEKxlnJkdebu/vLt7WvXbJSLbIXqtQ1reYwu6HjdgqJWlsmKVlGfcDqZq3XJLZXuQ9wTMVYzEiVbrXkGzoi/weIl0oNwHS6qmFKxfwWtPPbDp/qAW4c3ivg618fMVDpOylzgH87+0QSIejVnfAIm/nE5uVRemDCaYIiLyQ1ax6jd5LeyaIecZevM4kK++LwfJarRPTRXmY82Rtsr2m9o6W6jnzli3kaWD8bXgwZDRJBzIX4/PpxyuW3qCbKAoJFcrecN/F+cah+bU81BP8d4homKMnzXUBcfyw4fbQ28j28/3LEW1SKPvHnK1sT0czBe/gYGBgYHBGMKIXvyLFy+mQw89lOLxODU0NNCpp55KK1asEGXy+TwtWrSIamtrKRaL0fz586mtrW2YKxoYGIx2mHlvYLBnYUQv/qeffpoWLVpEzzzzDD3++ONUKpXoYx/7GGUyTEddfPHF9Ic//IGWLFlCTz/9NG3cuJFOP/307V5xAwODnQMz7w0M9iyMSON/5JFHxN933HEHNTQ00PPPP09HHXUU9fb20s9//nO655576NhjjyUiottvv5323XdfeuaZZ+jwww/f6nv5Mx75/N4gvdYGOcQCnQOzUFlaljORmSkF7nN10g0No7+hrj81xlrhy7kWcU4xjpo67/f8vD+f1rQw0GULLkQA0/SZiXXsTlcEvb8IUqalexxZw7RJRopmvgwLck6B6yd0uoy0ryAfuJZApLDomuH1T8yQRkUWNgObM4SIbWS9Mbuen/Xt6jpZBbi0LwPtBZtKk7kwcp/dDvopRGHMaG56Ti83RKQNogpq7j/PdHGmsBL2M2Y40/qoAFKt3cjtWF4txyP2bdzHAx/1St3N82+ZfhuIfLZERG/T9sDOnPcDyBXknAnnwE0ux8cwQpyrmVJYfi6XrEJ/T819CtYRG4a8H1zA7GlSh1U2X8MNgotbnvXwYLfs+Nw+qOVz3TDqKBGRLwsZJMPgQouP4Gl2CmnIEAjRB8mnudxBBklhf4NytJad718ZXvPyNVxXjCRIRFQCG5VCHVwwAxk6M9qALXO5jVmeGLb0KCQLop2G4tAXiutTrNLXOHDn60HDGN7v1xLjoZ1NGNypLb8cXH6MiAiZSh1wiyzUBPEUMW9zEPE1WiVdBSkItktgM4LJUX2aG2/p3SVqp7jz9fb2+9PW1PQbWzz//PNUKpVo7ty5lTIzZsygiRMn0rJly4a8RqFQoFQqJf4ZGBiMXph5b2Cwe2ObX/ye59FFF11ERxxxBO2///5ERNTa2kqBQICSyaQo29jYSK2trUNcpV8/TCQSlX8tLS1DljMwMNj1MPPewGD3xza78y1atIheffVV+tvf/va+KnDZZZfRJZdcUvk7lUpRS0sLlSM2kd8e5I+CbiJFcJlCeipXL2nCEFDoDkTWKockNRTq4u3eAtPNk+o7KtsvBGV9kBosJjGKFPAyZfn7KrQZZIAyUGfd0v0j7GO+qwCXCG8Gl8ZJ0hUo1AluNNP5Wf1d8lnLEaaukNK0ILKh4JaIBFW/JTiQKCLQyZyUlWZezQtJGswpgEsUuDuG/fKeChKxlCBqIro1Ik1PRET13JdeCJ4JNIFg1/BuMCgd6Ml8yuDys990TjjyZmt9ZTvUI0/qm8R9Xs5A8hnNOzU3nsdD0IEolfAI+XGy/8+rfr7/Hj6PFusPsh2wo+e9L2ORU7aoOi6loHIAkkK5MBlgjOoyDCkey90pllFqNFcxjFaJlLeCpFm5dinDYCS5UAeMUWd4GtnZzOtSpA3O2azPBVhUkJZOITcv5yZGnMPPOasg155MN0sUkW6Y9yDJqZBchw6Lv1PZfqXwgcp2SWtHdJML4HrTyA1RniK5aK+K69NX4jZR2icpuq1iX4Q3c1uVYvJ1ZsHcLENCL6eLXXDD7ZJmx+iPfZAkLZbqFOVCHMyVLIhoGoA2DXZKeagIzxrEtX5zhyhnp3jtwLGpenj8OFpCqoGxb20l1b9NL/4LLriA/vjHP9Jf/vIXmjCBwz42NTVRsViknp4e8eu/ra2NmpqahrgSUTAYpGAwOOQxAwOD0QMz7w0M9gyMiOpXStEFF1xA999/P/35z3+mKVOmiOOzZs0iv99PS5curexbsWIFrV27lubMmbN9amxgYLBTYea9gcGehRF98S9atIjuueceevDBBykej1f0u0QiQeFwmBKJBJ177rl0ySWXUE1NDVVVVdGFF15Ic+bMGbFlbylqkxewydXyDpdjmHAHrbohcYOWPMKG3McqyRHjhPU5EZWA9cFEEL0QQk2noNCqOrKR/8jCh46rRVnC5B8nNa2sbD/e0CDKBcqamfJQddDYTbRK9fUw3VYaJy30kcbEayDVZZVl+6AMoCC5RlmyoILyK9YyzR3Kctvnx0uKLZ/kuiLVPyHWI8q9Wc0Nix4e6MFQ0qICliBRSXgcRx90g+wekW2Wz5qdnKxs55qGt9BvCPP1XtjEOnU5D14KdZITDUAARKcW+OC3ZGS20CY+750+zlVegvb2d8trP5qd2F/nXJmIto8f/c6c93aRyLaIWjtlWzSAooHqn6+HpaRIu5T4LB9YRIMVvE+jSYfzCHGRkNAoVB94GeBcsopcUT2RCo7XckTjybE6MO9s2HZRZtTWIYzSKSz09UfN8X3RMtwHUpu/S1b8Z6uPrGwHUvx85aBu6c7buCbks9wvVpc8x84whV5yYZ5ulhVHLynlQ4t6gv1yMfSBWoT96sWlV4eoD4wzD05SZSmplWH5UiU+JiLtFeU5CA8TSjlyLKB3GUZSVZikR381KO3/98CIXvy33norEREdc8wxYv/tt99OZ599NhER3XDDDWTbNs2fP58KhQLNmzePbrnllpHcxsDAYBTBzHsDgz0LI3rxK93YawiEQiG6+eab6eabb97mShkYGIwemHlvYLBnYfQm6bH6/9m54emS8FrmO3JTmMr2p6RVKuaVVz2cy9uJTtTuyTzNhn9zkpCXgsyJBfok52eDxX4BksVHN3AZLzB8YIlNEM1FD1qxuo3p3Sakz5PDL8QYJEIkc8nKrvYgEg7k4KAABAIpNEk6PrSCrWFVFOgyjQYNQPvLYD68HdwsLe/zkGxDQQCLNzql/OEM8+gO5MHWx4wNCW6KBR4zVeB9YGsBfEKbmYK3SsMnR9mcA/kC6H2lJc8RgCbJpnjMBTTJBD1DZiSYtn87MXHIMkREfe9yjTl3+HkzmmGpIYJSEZE/C5Q3JMzxYty3KFPp8NKQ416TArN1QJND3yBl7tRKC203xPfNwtit7YHEN5rtIsqUFvyY8mUlb68gj7sLtDYGzClskvNZQcWdaq5ruU9ywoEkH0v5eOz50yAL1smxc/qElyrb9zZ+jMtpEiZKkJisJhjltdn3qmwUq8CLXq4Ebdokr+1BIKLIZI75UEjyGmXJZV/Q4YE0eoDxtTAAGxFRoA+8RMB7xApLz5wy/GkFIKBUDMZpTEpPOD59uN7UymRo5TpuE28NyCQRfoZyRHt1W9r/7wGTpMfAwMDAwGAMwbz4DQwMDAwMxhDMi9/AwMDAwGAMYdRq/G6QiAI02J0vDBrMONarfH7WVopJLRJWM2soPkg0U6zWRDjQRz5wCCc4mVPDkat+F5skTsk2QRS98eCGAy4Z4w/eJM7Z9Oy4ynbAxohsUnucPm5zZbvXB/YIUMypkm56BQhL5kYg6pNPCvGBHj4P9SJ0mwl0yNBjKgztBT8ZvYCmmTbw9WKgp/ohmQX55G/OcAe3Q3QV61rdfuma19LFOle+fWiXQjsvjSV8oA/73uSCvhxfK9Qh64PRvSKt7PaXa5Ai2qYUH/P7IbkKCIzRzVJ87IX2VkW+b7BHFBOaYEeBtUwIpEa9M+Q5awv9diGFwtZFWRxtKNQoskOK6mtk7P5ivB7+4E07zX/ka2TowypIrBJvYl+zvhapqQZ6MeIm78/X8f5EXM6FXA3buIS7IRpofOiEV0Qy+hzOYd0WQIG/ogsR59BVrDRFZmkJPg1R6sAGwtKi8FXDc3St4vZCOx/dkKYPblwOwfrQJ8ulpsEzgB5dA1EYuw+X52Re437dt5ZTPS+rqhXlylV8veJmrjf2pF3S3I/Rlqae51ywk/uoarW0Z+ibyOUcBy7gynYU9ljgNupG0f1yeJdNbEerT0apdEK8pvhy0F4Zvo8/rSWAquu/nju8+ZeA+eI3MDAwMDAYQzAvfgMDAwMDgzGEUUv1h7o88vk98ndKis3fx7xYMcxUSrnI23ZR8h1OGlxxBK0i6SR0B1nVzcd8kJfZ1lxGMCKUP415vXl77cpGPIWCkFt6TbqG6ylZe1rfy25kVZBEAxMVlbTc2WFIANQ3FSjubtnVOZAofFmIFAXslNLoeCFFIKum0Uv+LJYDF6Y894NKyOhZSMVhFL14rZasJZgYspyDzxCS8lAe5I8CUKSlt9ANRz5DuZ7ptkItX7sckQ/bEOXxub6diUcb3KPAY7O/fuhyGWGqsRTR6t3Eg63KD/WuwmeVA3JGeCMR7b7ufANJepTS3LlAgkJ5S4V43OjUsxViiroAbpxxjSb1ZzBJDyTF6ePt7rVSHpi8nrleN8DzJNDJ/H55unTnwrWjHMYc7qIYFZNAS3fBfSBSnpeRYyVXz9eLJVkmSW+SA3szRIAMggusD9xuMbofEdG+YfZNXtrBD9E7Va4pgR7Y7gUX2iYuV3pLSnfhjexe/VLbeK6DNnxtcHm0mrjBQu0gZ9pyzFgw0aKtfEHl8LXyWsYmB14VuTIfU3npzinuA/Ix1lOP3IfvF3RPpaDm9tcLibsi8E6Baw9Q+3xQ+/89YL74DQwMDAwMxhDMi9/AwMDAwGAMYdRS/Xa5P1mHTtsizU1gJWt5SH1ouaoD/Jg2WJbrSXp8oCr4fczLTIp0VbZXRyTFgvS8AtYoNx5oHp/GhdtcsCrAtFV3SF67uQooO8XUMyb/UClJE/mR7kTL+5DWJlvIMT+AQUl60FIcEkvYWpQ6ZaEsAdeAc5RfS0wBzYUWwcmw5EELgSTeqbKFngV2QVJsHiZRcZDWhfsPz+SJYz5HPmvUzwPALXGDqyqug0+jTpHGxOQx5bgmUUG0uChwkCjH6FET9w70R/jL+LUDuwk8H5HlI2rvkpRwA8h3gQAkwoFx5Gl9g6iK8jhSJK3/kSZHmRDHR7gpjadQIQER46CpS9UY0k3WYbjkOW5Ykx4gEQ4mGBNJfwb1L1j1A0Ut10Wi2ARYUzxeU/LtcE6VdEeY7Od88ShRbClhWW4817spBGucZuieb2Qvm+MnPVfZfuBVmdVRwRqKCdTyDSz92gXZjujpU6ziG/t7ec4WqjVvHrCiT0S5Hay4HI+izwIQsdOH65CmC2PdgtCOGtUfquf7uiG+rxuHiKjt8tXtDlRhK6e9+eI3MDAwMDAYQzAvfgMDAwMDgzEE8+I3MDAwMDAYQxi1Gn8hYVM5YFP8TenjhrqSkwHNGLWVktRWMDueo1AXlr97suMgKmCBdZcXuloq28JVTasPZt0Lt3HT9u4no6hhpikfCIRl6eEmXJr8GS6nwE7B16f9dgOdzUlD1K+IFH8im1h3s/YHzXML2Z0UZCQTrnlaFXK1vCO6kfe7NayL6tH1ijF4eHCpCvu0csOkiEU3rFK1bEhfHlx+uvgZSmCvUaiR1/V1cKS3YjXrbJ4W1QwjL6ocGl9AmT5N67Ng3ELWRIzIR0SUzbNIOf4D3bQ12Oz21zXrDq8vjmYoR5HyKYpEpdGFslnLzXVB/0IItdhGba3o4z7sSbGrWGOXnAs4XgVgdzws6+OUIOodZsGE6HGOZjdSjkJWOHDJ1Mth9sByHPRj0MetjGY3Ao9U3MB182fkhO6L8LEw6PqYyY46ZCjBX3QcOeR9ohtlO+IzleLceO+8ze7MobSsT/gdtp/6W9vUyrZPq7dIkAkavx/mlhuS/YiufqUwtHeOx0ywRxsLYO/R083zPtm3QZQTLtBdPVy1Mo8z/T0U7uS1ogwRGa2iXOPyXew6WgXPkGkBWxRtzAzY/VhG4zcwMDAwMDDQYV78BgYGBgYGYwijlur3FRU5SpEbl7RTMQlcBrBBXhCpcM3lLstUipfJwn7p9uXLQrQpiATYk2MXnaD2U6kMFBJScYUkb/sTkpcJvM282usdTIPFWyU1tAEi99UEIcpWDt0YZX2QBkMXPkuL8IeuengNXw6iFKZk1EQCSgoT9gRkPhWKbeLnsDDiYCdTryLhDxEFU3zfQCc3ZHqy5urSydf2pXn4htpBFtGSC5XDSItCUhxwfYyvlh2rYJwkVnLdso2y3NtddZVtTIiiwMXR1SQlIY1A8iTPL6djCajh53onV7YDHOyMchO0sf5uZzpby/mNMlieRZZnUaEg2yKOrnU17O7k9PEc7p0qo+vVJXn+oECTadKjtQG1Hhla62rvlO5czZBIxwF634Z5pQUfJBtckSOQuKl7phwf6H5qF+HawAjbdXJNscs8nzDJjx5pEtcHF1xgYz1cJBWWY6c+wPMWXUlLWlvla2AtRJfeOFe8FNPc0GpZ/puRZK3rWdUkyvky3EYlH6/HdpmvbWkybKGa19nYBugXcPu0tVxWNqpF0IHWxPGiXDEB0nKM5RMhAWguy6UoSLQgP1JJi/AXhkRk1ajBoJu03q/997W24JaMMF/8BgYGBgYGYwjmxW9gYGBgYDCGMGqpfievyOeqQXQJJs4oNDAlEm3gZC6WiohzkPpyYjJqFwIj9xVKfN+Qn6kYV4s8hfQ+RtYCposczRLcBfbaw0QQmkThumD5iQk+omj1K+UBJ8+VsMpD0/5ERPkGtowuQxQqpBlLzZI6tUEyKSeYWsS85URE8XXQR9WQnxws6t0qGS4wDxG0BtGTohK8WayHCGclpsQsT4s4iMEMIXkSUpVlbVhYYa5fro5v6slAklQXYdo5m+U2KbvcDygH6dewCmiqrdUBujYP17Mh6XagQw7IPre/X7Pe7mnV74UUUUhRKKR7wvB2ERLuuHFoZzntyevjaHsY7U9vZ6Rd3cDQNLmXk0uliNJZC1Hvyti58j7len6mQhVY69va+gDSVGATL0qlGEiOQa19HKD6q+BYRta7po5p+/R6Ttgj1q6SrPjKTAPfN8Xjqlglr12OQXsFebuphrXA1s1y3uPa3F3kNUmPJoqyBIGklq8BTyNPtiPOM+xXjBqKc5uIyMK5hW0MlvtERCGQaBWsN5iAK9ssH6IElvz+DIzHgBadtpX70g8JpTyIaOr55TkDHmHuVn7Kmy9+AwMDAwODMYQRvfhvvfVWOvDAA6mqqoqqqqpozpw59PDDD1eO5/N5WrRoEdXW1lIsFqP58+dTW1vbdq+0gYHBzoOZ9wYGexZGRPVPmDCBrrvuOpo+fToppejOO++kU045hV588UXab7/96OKLL6Y//elPtGTJEkokEnTBBRfQ6aefTn//+99HXLF8tU1OwKbQZi3xAsTusIC2zaYg93ZCD27BJ4V7mEYpxTW6BKhCpPZikIilMzy01S8RUTkKlvKYuKYk6xMAJnpCkk20uxNVolwoAFb0Pn4Gzw/WnVn52y3YAwksIGezreX89mfAGhZ4S2Hh3yETkxBQWoE8t4mTl5RWjhlEqoWgKkix2VkZbCXUw43vgNfC5i7ZJhPzECAFEhShdW6pXvL2mMs92AmSAkgcpZhGE1ZxfVyMFyOrTWUProe0fQFkGs3AHq36hfVzTtYh0AueHDZa/4NckZAXz3j94zvnaQnN3wd25ry3Sv3xjTJ9ckwhi+9hci4IVJR4W3aOl2WaPAfrQ6JzeI8HDNDlTwHtP14LxtIGgZtAHguu4YA0qUnjxDmhNTBeXfQEkHUIdsGcsfFZuUx+tfQysDCwVREo5R65PnRFeD6FgNJ3ShDkKijbZ3p0c2V7rW9vvqfWjMEuuB5Yt29cywtCuF3Wx9/G69/GNHthhNrltV3wavLyIMN2cFt5fnnt2AaQTPqgvSGAjyedhsjHijHlwcuLLHltEWwNxiAmCwu36Z4X0P8gwyrNmwfX91IV3BckIV3WGvAy8PJbkEkBI3rxn3TSSeLva665hm699VZ65plnaMKECfTzn/+c7rnnHjr22GOJiOj222+nfffdl5555hk6/PDDh7xmoVCgQoEbKJVKDVnOwMBg18DMewODPQvbrPG7rku//vWvKZPJ0Jw5c+j555+nUqlEc+fOrZSZMWMGTZw4kZYtWzbsdRYvXkyJRKLyr6WlZdiyBgYGuxZm3hsY7P4Y8Yv/lVdeoVgsRsFgkM477zy6//77aebMmdTa2kqBQICSyaQo39jYSK2trcNe77LLLqPe3t7Kv3Xr1o34IQwMDHYszLw3MNhzMGJ3vn322Ydeeukl6u3tpd/+9re0YMECevrpp7e5AsFgkILB4KD90Y1l8vnL5EtJcdrfx+IG6qt2I+s2TlE+lr8HEspk2f0qkJIuMdENfF5uKmslQR+LcP6MHjGJNwOgp7mgHxfT0pYApahVm1n/auiTolkWfAedPGrBoPEmpfZo6f6G78LWXHRQO0TNGV3FVFC2owVJJ1SQn0nXvYO90EbgrmP3cdvrkfvKoOFh26ErEBFRKcaR8jyMMNbNz+3fLG0Tsg01lW1MxhNbz2WcvBbZENoBtVU9IVFVkMdnqpo15bTF47QUkUKiDddzEhDVLC7bu1DLzxd2uBy6g/q0pCczgv1ZkTLF7Ru5b2fNeydvkaMsKmlh73xg/+AL8nzEMZltlMIn5nVB3Rtdu4iIfBAFzYUqYfurghaFDUxPymG+dmAj96GjRYVD7RY1Z33OYvIwB54Pk/mguxyRTFbjxCBxTZ+2HsCwwHmGzW0V5SAvgK+f0KY1c6cSmB1YYLfgj4PNgqP5wwL2SnZUtl9INohj6IZrx1Cj57piRD4iokwz/L0RDWsg6t6gREp8Tl0tuz5aITlWcWxYIbYfwXXEC8q2x3GCdQ2GtDap50q5rVrmtsp95N8D0VxdbR0bDiN+8QcCAdprr72IiGjWrFn0z3/+k37wgx/Qpz/9aSoWi9TT0yN+/be1tVFTU9MwVzMwMNgdYOa9gcGeg/ftx+95HhUKBZo1axb5/X5aunRp5diKFSto7dq1NGfOnPd7GwMDg1EEM+8NDHZfjOiL/7LLLqPjjz+eJk6cSH19fXTPPffQU089RY8++iglEgk699xz6ZJLLqGamhqqqqqiCy+8kObMmTOsZe+W4M+VyVcqD0oUE+pg2rYPEiW465nmC3VpubMz4PLRye42NE2626ArD+aEtyE5gh6FDSlITIQT6ObtYrV2DjxSvoNpIoyaR0RUhEQlGJUKc1WXasQpgkL0ZTEqoCyHCYpCkH8bJYVBSXqAtqcyJMvRitnoGgTPpBwMuyd5UIwI5gGrprvzTcgjhQ1UGnSd0qI9ClclH/QXnFPWGDUvBPQmSBm5Jkmxxvw81qpCvO1Uo9whqX5MdOL2QsRBPdgeNPcH42sr2y/n9ufztbzlG8r90Raz5e0XuW9nzvtyVJEXUuQPaolLFLdTKcfbHtD2fi1JC1K65Ac30JKWFAf6A/sao/ORFl0P3Ud94OaqwhCdUlMyXIieWUhgNEg9sie443WDvIbVjsn2weQ3HshepEXsjEKCo0IXtuPw1z4sxslz/hrgH3NIixNp1D/U1YMDXlzWByVDjE6pNPa7MIE7Jhrl7XKEXXdtTd7C6H9ifkOxkvSKJD+z+1SGMHgqnRHlAt1wLMdtimOJtOHoFGHtgfXcykmtVIGvnh9v64F7r+a2V35XClGavDQcRvTi37x5M5111lm0adMmSiQSdOCBB9Kjjz5Kxx13HBER3XDDDWTbNs2fP58KhQLNmzePbrnllpHcwsDAYJTBzHsDgz0LI3rx//znP9/i8VAoRDfffDPdfPPN76tSBgYGowdm3hsY7FkYtUl6Mo1B8vmD5KQkF5OaDjwNWD96KUh6YUkKqlTNPE+gjq3oXUdSfukJ/PfkyZv42sC56pR5ASIrIUWN1Jm/WkoPhWrmg6bsy/fpeVXmfI5F2XQzPQ68GSDKnD8pzTv7Wji/db4ZeB8tURBanJb4FMo08v7wJtn2dp4pQKTCS5KNJ68dIp5BwhE/WMa6ccmDFmPQsH6g/V3Zl4Vqvm+odRgTFVvut8Fq24uhlTRG1xs+WQfSujod35rhh+9Kcx9ZoCPE0rqFPbeJLwkyVEBOR/TeCEIlco2YBEleO2n36y5+e/tR/TsTytf/LxqS9GehiindSBWPeV8veDtMknqNUy2TTA0AE0IREQX6QMoD+holrKJGa4tIchBJ0Y1AAiG/dk418rAgJRVkOT94G+H10Kpf5TVrfbiEU8MFrTWyTYrFoaN04rWtbsmzL0vvBecMH2nSBeUUrz2hrqeyvc6Tz5pvYd+Lk2v/Udl+oXovUc5Kg7cEWtSXh5YViaRsg1bwZXgfhDq1Z4B+bYoD718vNVX0iLDCfD2REMyVcxM9l7DDrJx8PyhY8/D5LFivStp4HHjfeFvpzGOS9BgYGBgYGIwhmBe/gYGBgYHBGIJ58RsYGBgYGIwhjFqNn5x+dxqRdYqIvCimqEK3Fci+VJJCh6+bxTovxbqNv0v6oZXirNU0RricDwSrjXryI5Ba0C0nClHhmj+ymRCvdUysbO+XZI3/6dAEUa4hyvXrK7NeWWoAd5s+qZVbtaAJQQSuuqldopwbSvJ2GF3PoIwecQ40fjfGx3R3JHTrwUhfmEnNyUm/k1KM9XE7zPeprZZR+CwwpMCMej4RzVBmdvNAW6sb18PlomzvgW5+RNLFRkRpi8pyB9ZsrGw/1jWjsu2iK5AWTEu4Pwa5HXxaVMhSnE/sLrPGjW5hKiS1/C6332Aj6+2eGr8b8UiFPTqocYPY/3o5WdlGEx4vCnYjWkQ+BUmA4rXsF2WXpIYtotaBbQe6YwUSUof1/OEhy2FETE9zSXNgXBdj4Eo3Xtrp5Bv5mYLdPD7ykPXSqZI2EHaJ52MZMotaMc0GJAKJkRweUxCcj6xGWZ868HHDNvbpGT8hmyS6Qm7qhgnUIder0GpOw/evDOdr8PcNrWETEWWzfI26IkZAlPPHl4HoosJmByIbBrUxA5/C6SLfJ+YM/42M7nwuRhPV9HZ018ZokeTTssnCew0zN1pgD+QUpO3GQDlXi0Q4HMwXv4GBgYGBwRiCefEbGBgYGBiMIYxaqt/JKfKVFVlFGUUqtAEiPU1mXsPXxlxQsUpz68gD5RNleqtcJSnh6HqmX17eyK51Z+79fGX77dwMcQ56TSG9hS5X/35LuulFNnLBtRl2E4m2SW6orwCUXwoStqyCZ903J86JreEu7TmKqaHOFbWiXAT80pAgwyRESIkREVGBr+eDhETKke2I9Fmgnetn5cF1TZMRAimg7cGdr1CSQzSSBpouynXAKIWB9d3inHKUE/t0rE1Wtsdh4qO89hsY6DdMOIIRu4iIOoo8njCh0MZOdlPSk4cgfZdrZV9Kn0Y7IqVf7WOqOtgJ7mMBSRPODPZLR+ntnKRnZ8EqWmTZFmXKcnzo7kuV8rmhkxcREZEDSa7Az2pL0TdxMmBimHynHONICTtA25aBwlfa6uoCBY+JrLyM1ATQZdTOQYIwUL1cS9ccAeAyh89ARNS9mV10g1kYR5D0x1onaeTmWT2V7QBE2MzVyrlQgkiqGLm0qaa3st0VlBKfl2CJr6fI99VlEhek3IAP3H390N5a/xeTvB3u5G2ULH1atEccGxE/r1dWSnLodrGej0GSHn8vuAhr7nz+HP/tBxdfr0PKsL7eZv4DZa0+jPKpSeDvuo7q7ubDwXzxGxgYGBgYjCGYF7+BgYGBgcEYwqil+i2lyFJqUMIVhN3D1Ee5nikkSzOjRlpE5dkU1clI+iZfy7RtTZyp1c4S7y9pNKELf1uQDMYDK/FAXFrgFpNc75CP610OaXQmJreAKGAiT7ge2a4GLGs7wYJWs7zH5CY+oPzsMlzb1urj5+HixiCxT1aWi26CHPM1kKu6l9vUKkgZASN9lTogEla9nngFtuFnq7CU92vDGq5tQQISXx4T8WiR+zIsUaD84Wr9ny5xO7RCQiEXaDlPG8KYMMSBqI75OkmxIv26MtfIdUVLX43Rf7jvgP5rpUtEtJF2N/gyFtmuRat6pDSFfZApQ1/ZW7DQznPbliEyYzkhipEP1LJyBKJORiBCphZ9swheKCGQxDBRjD+tJfbp4/GGVt2hVi1iI0hqBHMQx025XUoPGDIOvWJUWht8EMGzWMP1DnZxObdJmuu7sA6hwuDIZY18afCsAdrfZ3ObpLqi4pwmi9u1LQeRQrWlB6W4QACeDwN+9klZ2C5CFEVY7zAaoi4poDyj8D3iymujDIeRQtE7Ql8/C3F7yGORWhlhspTkfilF4ILwrDhOiVhmNpH7DAwMDAwMDAbBvPgNDAwMDAzGEEYt1a9sq58OsXT6G8vwth0EK3VLPhZaVTtRSHYTkjwPUldtQNuePp2t+p8tHiLPAXo/zfEnRACKfEGj8uAZMMmLTgnH/EyZK5HLma/d0iQtQvv+xpkyspg0o1rycp4fAn6IQDggV2gW41QCuguSdSidyo5wx4Q2Y6YTkGNC0gTXlweOCqhcx5HcFfalVcJIQcPUk4jcMFBswNhiYqBCraRlS81Mv+XrMGmGKEY1QZYvWurZm2B1malqQdeRpOddkGMiHXKsF8tAB8LgxHHi02SW/cPriIgoW949A/gof/+/YlkOqigYgxf6MNkTeHP0yT5UZT4p08s8eY0WhUuOeW5PfwbmcJek1mMbwZsgyOPID9bfGFSHSAaJwuRexaQc4x6McaSEUZIYFBUKAp3hnNGUQIommMbPdzK1jrKEKsq273XBEwrmku5FgQF9cJ5u6AZtpSC/NZ1unj/pMiTi0YIDISVfhrHhwLpRjgwvCzuwfqKnRCmmedzAOOvO85ip7ZOeQkHwWvB62GvBdvkloHyafIhJkUASUnoAnxKfF20FmeQgntO+nDxnIDGdpckvw8F88RsYGBgYGIwhmBe/gYGBgYHBGIJ58RsYGBgYGIwhjFqNP9tgkxOwKdArXZxEQhjwLcGoRvka+XvGn2Uxyq5mTT07ISLKoWYVhAhTLviW6Hq2cN8A8T43CcTg7PAa74fq36lsP5VpEuViAdYLe8BVCaNxbQRbBCKiBGhUTgFsG96W7WhBEhfUiUsQDS/Qq2mmUdY5fZ2YPEe66KBbFSbzsbNgX1Et274EdgHKzw20b0ObKLe6afqQ5UpYBb/uHgVRD9eAiw+MJUwwQkRkF7j/gmBGUZbNSJvBBWl9Z5LPR0MO7ec1XkNFwHUnJgdXsZrrvTbHER7R5ajQILX8/LvZTPK7aZIeZfX/c2ypewtXV7B98HWxRqz20joHdG8LdG9HS2TiiDkD/Qa2NHoyJLRjEXYfWlIxhC8Fc6sPNF7NjinQy0KtG+bOxn63auVDKIfnWSnHY9zRhkGmg+ddtANsadCGKCcHbK/L7VoOgT1DRhSj1LSh1+Yp1Ri5T4rQhRa2pZkU57WwJyfXQnSfy2f0EI3v1k3T+NFmS9hN+MEGqUuz7QHN3w+NZ4U190mAFQADBBy2mn2FSISE/dIrE5HZJXbdLSTgIqD96+587rtmLyZJj4GBgYGBgcEgmBe/gYGBgYHBGMKopfqDKUWOX5EvoyV1CALNAz9bLPBbibTJc5w0Jltgfiq0WVLU1jSmwZAAeqjnA3zLsqSGkELyp9Gth7fzE7Xc2eWh3U70BAubM5zAJYjMMTye7ZOUqAuMVDkOySw01xIbowwC5S0ic+Wl75qVZR4JaX8RxYq0aHIYNS8H51uy7QOQtMIGl5+1KRnVKtrKz9QLUcnCUhEQQHoy18zPFFsPiXj0iFcQjcvysPElxbZXvKOyvTHFskuhgAk15KUjm4G2bIDc6RpLjBERfVBB7GPyy4pH7f42tuzdk+on1f/PsbV5BitVsFrz9XoXvrzWgEChe+AiKnKhkzYXMKESugv3yaUS6fhSDOh4iDQ6KLoaPJNwRdYS6ZTB/TPQyc9qlSES52aZ1x4P4UBy8lpee6C5UfJAqt/RqP7WArvjRVv5pK59JP0d6kBJFNoeXA+7NiTFOY2b2E0uWx6awu+/CG/6AiBTpod353PA/THcCdEVQcYrxmX7YPKkVI6fryoto2AKd0NYH2x4D/m0xD6BND9fsAveSZo7n78PE6BBv8TAvduWruiBd/ODuTvDne+6664jy7LooosuquzL5/O0aNEiqq2tpVgsRvPnz6e2ti2sygYGBrsNzJw3MNj9sc0v/n/+85/04x//mA488ECx/+KLL6Y//OEPtGTJEnr66adp48aNdPrpp7/vihoYGOxamDlvYLBnYJuo/nQ6TZ/97Gfppz/9KV199dWV/b29vfTzn/+c7rnnHjr22GOJiOj222+nfffdl5555hk6/PDDt/oewa4y+XxlQaMQEUU2MZWSmj50NDudtvUCkIyljy0o3dA4UQ6t2LshT/qKWray1BNTeMO0oBcAi/HNkpZBau/Bdw6obFdrFGRvjvm7eA/S7kwZ5fok5ReGZw9A4g1dRgh2Mg3lyzDtrjDZR1qjVMF61YLEJDqVjdQpRjVTYG3v75QmwaUqzGMOEc5KsoGDEFkNn6kUp2GBltr+Hm4TlG0wYhuRpESRtnS1pB4rUg2V7aEzxktphogo1QIlIfmLb5WkTj0fl5sECcWfgWCIgVZZoYzXPx6y29mqf2fMeSIiN+qRCntUduWADUOSntoqHjulBqahy1oCJSfB0kukisdyplHOGczJLuh5jBJaIyc+0vuYAEZEtNTmnBsHS35nuNFC5AUwpzv3I0YmzI/LiXOcIo+d8eOYPt+gakQ5fKa+yeApBBbjSpMP6wN9le1/g5cBjk8iolIVrB3gUWTDmuLv0mROkNTyLkgmWtvhmm6Dh0Y5zmuh0tbiEjg84dhwo3yOqxnrl6Gb62I8zuykzOyUr4N3TxW/K1A2zTdIzyWMPpiDMehrk94o6Knjy3B7edBHtvYeGlBjdqhV/6JFi+jEE0+kuXPniv3PP/88lUolsX/GjBk0ceJEWrZs2ZDXKhQKlEqlxD8DA4PRhe0554nMvDcw2JUY8Rf/r3/9a3rhhRfon//856Bjra2tFAgEKJlMiv2NjY3U2to65PUWL15M3/72t0daDQMDg52E7T3nicy8NzDYlRjRF/+6devoq1/9Kt19990UCg0f0GAkuOyyy6i3t7fyb926ddvlugYGBu8fO2LOE5l5b2CwKzGiL/7nn3+eNm/eTB/84Acr+1zXpb/85S/0ox/9iB599FEqFovU09MjvgDa2tqoqalpiCsSBYNBCgaDg/Z7Pps8v01OWooW5RCLub4UuM7UsAbu19wo7CwLIqrI26FNMmJSajLoYXEWZld38f6EFpkrmAJNB+oTgwxgbZrMiW4vhHpVSNPMikN3TznK1w6sly4wxSpwL2RpjrLNUvNFlxYfyO0+iHJn5TUhKcc6qaphzcuflvX25fgavgzfx8KseVoGvUICtD9we0q1xUS5cetY4A5P47GAURdVUHN16eHnKFfxWBPuTJo25uvgsVGMgw2EpvEHHX6OTGboF6M/JzVTfx8/az7P26FOPWscb6/K1g15bV0L9b8bRs6vG15sI3bEnCcaft7bBYtsy6JEWNqXYFK3tnYee5PAFkPPFudluAFFlkdNXkf9WGxDE0ZicoBYkCLRBU3ejfAAGWT/A2tKoQozDMq+8mX573I1678YEbOUkg8r3HAx6mFJPuy+e2+obK94eSIfwIyRSTk362AhydWCjYyruUWCrl+o5QsmAtwP5ZgWkTHBc2ZKbG1l+53oRFEOXYabk1wfz19f2S5G5WRAd75inI/F0P1Oy7aJ7pyJAI/BkhZdUbhgoh3aMBn4iIh8eXApzUCfB+SighFJ0UU8CnYqpbhca9x3/b09kn0yHEb04v/oRz9Kr7zyiti3cOFCmjFjBn3jG9+glpYW8vv9tHTpUpo/fz4REa1YsYLWrl1Lc+bMGcmtDAwMRgHMnDcw2PMwohd/PB6n/fffX+yLRqNUW1tb2X/uuefSJZdcQjU1NVRVVUUXXnghzZkzZ8TWvQYGBrseZs4bGOx52O6R+2644QaybZvmz59PhUKB5s2bR7fccsuIr+OGLLL8FpG7hch0dUyd+Tsg+UpYUifoeuYV+RyvTksUE2X6JhRhenh8gpNMpHzSbyxXC/QbuI/0OUzReDE9ch/TdNkcb4c09xjLxqhUXA4jO+Vb5LX9EB2qcCD4fXVJWrUEFFtuHNNDYUjcUWpKinOcDNOdboyvV6iR9FKuDtwnwWXO14P0pkZVoZse8K2xBun254b4GrlGLhfoBheonOZ6NQ6oenQVBHnB04MpArWHyUhyEY3eBI51Qj27UXVmIAqkJWnZMg47SDhTksEMqVDNx/aLceSw5Qr86DXausftv3jO0zjMHYjtNeeJiPy9Njl5m9ZurBX7m8El0/Fhv3Nf+3KynZ0GlkfS63hy6m6zHpyGVD/KP5nVur8oULUY7K/A+3XXX7udbxTp4BvlNshlONfAYzm+kmltuwRzRncV9HMlOnqH92195y+TuX7gcmyBhGn3yvo83bV3ZTvcweMqXzO83BBbwxVcNRUkVKVR5iA5bsgmK9tCDiWiAlxiQweXm5zm9Txaku+K9AReK4IbQMLcgiuyH1w7O3I8IRNakiZM1qaquBzKnsFWKSVnxnEUUgdof+rsEeUCbTz28/B+KazmMZzQZMEBN1S3oC0Iw+B9v/ifeuop8XcoFKKbb76Zbr755vd7aQMDg1EIM+cNDHZvmCQ9BgYGBgYGYwijNknPQLIOcjRLTTT2hYQIIsqWRpkrSFDj1DON4moJV5CC9SASYBlMpz2/fm2MJAcRqoDlsQpannUIAvXBiezG9JZvH1HOLQ2dzKcEVv3hVZJu8wG7X1jHFsH6lfzd3JBOnikxjC7l65RUFVrlW22cpD7YvZesN0SvcoD6xAQmlibh+HPQgUE+FvJLyjrbBDQm0IblOJ+fmyIT+4jc6UC/Il2Xb5RUXgnu44JK4mgJVRpDTMW++q9JfO0oXy8ckWNY0IsgPQyihuHRbeCgPc2zAOG8y7c6esaf3QSFOpfssEvNTd1if7aBo2finOmMcZvrVv0qy2bdE/fh3AGpf8uInQrWgRI4kaCUFGzWIk1GUMqBA7BdBOmPiMitYVo61cKDqrRvVpTz3mZK3wMvgWwjX7xlYoc4Jx9mDwo/zBm3Sg4qXz3fK7sJPWZgjauSc+4/Gp+tbH+3ZnplW6fJ0duomAQr/AA8t0ZFY1TVo+verGzf6Z8irx2DiIiQcKkcGz46Kf6NUogXAllYGzNlVCBdWCA8GX4Tk+dgFMbMZG677CRtAAAKCfAy8GtW/fBW9sH7LpfkOpTDUrodiCTo6YmqhoH54jcwMDAwMBhDMC9+AwMDAwODMYRRS/W7IYsoYFE5IRMYlBJMZey1F4cEfWsF03d6XmbPYf4miOahW8CHJ71T2T4isbKy/YPYJ0U5lAeQNspM4O3EBPYKICLKdjMV3ZZlSllnZw+YyJbcnf7JlW2UO8r7SzrefonpO2si03rlomyTUg23Sb6B6anQZh4SpXGSqvL1spmzSvJ9suMkbR/oRQt9SNYBSX/yk6XVtqDpQDJpjPWJct3EbecCJWl3cb3D62Tcd3c6aCsJpDGZLgv0yN/Ado7LYd2GS8pERDRl302V7bYU96vtypNEwCOMA6JR+EVIelICXQJlpJKMb0QnRPpp8D5NStldYBdtsm2bssXh9Yz2HD90AMZkOSLXClLcBmgJntCSJgXSKKNAkJUsWFRvkh5AOFd94CVQqsKc63JCFzrgGCQE6+uUtC1KB2j1jtR6X16eE+3jZ2jv5rra3VqCsKm8FgU6eEw5SBGX5Fz4Q+dBle1QF9fH88lrpyDITqALvIM8kNpapKxRrAYvJJgAemIrDIyjCkPT+05BG/PYjuBdhFb9euAuTIRUArlX5WXB8GZY19p7Ktu+FK9PoU1yHco2QNKozbC+ZGSb4LP7oT5WmtcRR8a3olD7yKz6zRe/gYGBgYHBGIJ58RsYGBgYGIwhmBe/gYGBgYHBGMKo1fiDXS75/O4gty9fH2sYb61iF59AtwPbUjPBRD92D2vGVq3UBDFCW3uedcRiFTdTqEfWx25nDcYNQuKOIGhFrtTXbYjWFg9w3Xq1nCWNYdaIOjBSFGyXumUEvFAJj8EFA5r+BZG6/D2g9WFUMy0JB55jeeDq1Cl/PwZSfAxdXRS4Zjo56TJkgQ5uQ2S2Kr8Uszogwp8F0bTQrqNUI/VYdOXxBSGZE0Q5VJprJyZbQT1Xt8PYO8p2Ji93NFe2YyHuV8uVkdR0XX4A5bBWhzC3QxpCVqI7KLpQERGtKfe3Q7q8dVrfaIOl+v9l89LPKgnJsNB+YgLMBaco28IK8/wOR2ANcDW7IYjYiePfl+H97gTNLS7L47UchiQ9kEhHH1NekMcrrgGW1leBDMwfcEUO9nCZQkku3SL3DWjT+njN9PE48oErqb2FnE6dBY5M52FCIs0VTkTPhOuFfDznSmnN/TjHBV0Q7F0t3xXaWwSnspGLXea6uUG5DqEtDK5DXgDdiuV9SjEuVxNhd1ArLCuURxMliCaK+jzaEgwCmvkkpS1VoQ5cgVuhn7FfPa1jB2yptnLamy9+AwMDAwODMQTz4jcwMDAwMBhDGLVUf2qKn5yAn2Jh+dukWMOcVjjJNHAxzPRIrkmjv4GK9uchJ3ZSuqMUgZHFSGmzQqsr27fWaFHYwE0kXw9R6iK87dNYGYy894mGf1W2fxSeKsrVgvZQqILIWiAjTN6rTZzTtWI83zfJ9KTbLXUE9GrExCSYsMQLSonCVUBpgZuRTi8h3Slyldcyx51vlPVBShITJHXmZeYajGaHEbx8eaDBNHkIKeBSH9+3GB+eF0NKswgupLaW33xVjvOBI6Xpgk+W0vrfDYNkAvSvpfnzOfBMq7LMLZYgSqHSJJzmd+vQ59tN3fny/VEmk7Gc2F9I8uQMQDsX6ni/7pqlSjzX+7p4HIW1prFL3J4eROJEGcUXkLRtrg6i68EqWrWGx24pLpdXC6haEWlU+/zCnPDoHojlWqp7xDk9cW6HqlqWCFOe1JUiMLfyEP0Nn9uKSBnuyom/r2x/se4iLqe1YzHJO1AmCYCOYGc0V2uI8vlKiqUyn+x+EQUxvYa1rjriuvqyso9cyHMf6uCxgGuN0kOawqNv6uWbTsxsFsVQdvGqIIojdLkbl2ucHyScQhXfONIj3f4CndwOmDyJIFogup3231j7/z1gvvgNDAwMDAzGEMyL38DAwMDAYAxh1FL98bVl8vnLFN4gk2ME92GaJ+eDKHXIiOSGt6a0Ckx1BXolpVW9kn8HvXoARwK8J3p4ZTuyWfJbZaC1fRn4HZXm7WxZ/r6KQdSnxzpmwv1lfZ7tnAz35WfKNHO3rdkkI+BVQfVciNaHSSWIiEKdQEuHmE70A0UX3CgpKKvAdJkXY6oTPRiIJB2P1qe+TZx4JewlxTmpvZiKzfay1XWwrlOUK6S5HULrkAYFL4VWGSkxX9/Af0BVo9CX5ajk/Pw9zBuHNvN9ytJhgEoen5cMMT/ZmeOCbkDKA5hr3JsO8pD2M7yU4PrFfDBuu4GO1jwEYlY/venpPOxuArtokW1Z1N0nG7oW8tf3gkV7sJPlvlJM9qGFyU9ggdgStY7NFm7j/X3jpAwTgEh5+eTQ30/BHjkvsszgCvlJrBtEFOjjCRTo5DHlFLgO77TViXPq4Xp9QIWjBEZElPW4XcPdeF+YP+slRX3l2pO5FEbK05JKBSFaXwEk2d4CrxXKr3HRsD7Mrl5V2X49vLcohtFA8wGQ3opbmMMZkPhAdgltZM+uQEp6eOB7JIPrdr1M/IWJu6xWXqPsApez85rnElxbeFEoOVdL0HaBFSAZgmase2sMqITeVk5788VvYGBgYGAwhmBe/AYGBgYGBmMIo5bqLyQdKgcciqyWtH0ecmSj5bzdyzSPT6P6nZTGSb0Lqyx5kbbDeHtaPeeb99lAJ4UkdZYej8E/eH+gh7cbPihzZ3du4tzZfUWmwTJNWiKdNNPfVRCEBq3CqVsGxEDPBJXHfNKiGNkFCJwRh23Ij60C2vCAhDu47UYk75Srg6AjKbDqr2ErWQxMQiTpUruH71soyzo4OX4QFyg/QZ9XSZoYg3eg1W7vZG4fncJHzwCM96IHFjkovrayfeuGoyrb+TzTspojCPmAgiyXIPBUr2zH8EY+VgMeHkhHOzl58VXl/r5Ml3dPqt8LKLKCiiIhOWdz4BFSLCDtzh1ajMm2UMB7BiIsU7lBOWcwiEuxCnKrt0AbatwqyjflCCRsgXlVSMq1QlXxM2UbmCsuJSUlXI5gVigecN37c32cdZKiztUBJRzn+3hagigMelWoxbEHSXWq5PrZUwDpDZLGZBt0DyfYhktMiPdUtjcrKU06paHHqZ6kp1AL9H4TyztukJ8PPYj6j8GaAM1QGMeLpN5Hvhzfp6WW6211ywoVqvk5LB9fHD0bvLBse1zjhExSr7UJyMT5avRWAg8yqfSQ966U6+kuZMPAfPEbGBgYGBiMIZgXv4GBgYGBwRiCefEbGBgYGBiMIYxajb/qnSz5fB7ZGRnCKbSZQ85lp7Pugkkz/F0ySY+d4r/La9ZVth3QnImIql9jLWvzdNYU50x8q7L9j77Z4pxwB/92Sk3j/ZggYmOrdAUBmY3K4B/j6VGkAMItLg2RveJ6lDr4A6K3WTnN1QmuF32HNVN/ls/Bduu/GYt4Xn2ysh1qk78fI22QwAe0MWcT2zrYtUk8harf5P7rOZDr2pXT3Lr6+AGdPOukqClaOakPOwVohwgkAFrL7dhxoKYPRlgHdmAI5hple08LcESvJCT1CCfYFbIzPEGcY0P1LHsLbmYgZe8V4giNxWqI4BWSmt7e/v56p/y7p8ZfjivyQoqawjI5Ux9EyMSEO/l6nqda4EOywA6lKsrX85SMBolaMEr5frBPwciZRFoUNtCJS1VciUGJZiBKI0aMs2JS48fwb06ez6layeO9b7ZcF4Ov8DEH71Mv9fowRO7L9XJdbXAH1F1/Z9evrmw/GR5HwwEjXOJY9iCKJVVp4j3o1iEwEihUyzrg9Xx+fCbU+LUkV+ByVw6BDRHYYZRk/ixhp5UrwboYk4MrDC7ZqorHk5OD/Vp0PbQf8OXBNS8t11k/rO94DsW47SIvaxFt37VNcQtbl6VnRF/8V155JVmWJf7NmDGjcjyfz9OiRYuotraWYrEYzZ8/n9ra2rZwRQMDg9EOM+8NDPYsjJjq32+//WjTpk2Vf3/7298qxy6++GL6wx/+QEuWLKGnn36aNm7cSKeffvp2rbCBgcHOh5n3BgZ7DkZM9ft8Pmpqahq0v7e3l37+85/TPffcQ8ceeywREd1+++2077770jPPPEOHH374oHO2hFxTiHz+ENkaF1OqAooEWI1iNVBvDZLK8wNtiw9cSMoIVdkmoGkgOthjPfvzOQn5WwlzedtAE2UgSlcoJrOHeAGuT32YNYFeLeAg/imSh8SA/klI6sxazRyiA8k2vLzmKgjJP5ASQ7rMrZVtbxX5epbLtStr7nzoPlcE6jMQYSmlnJTuSH0t4GIFof/GxWX0wL4aps3zTZCgIw89q2XFQRlAQe7zErhN6TnRUS5yIa87Jh8hIlqa4siLXWmWJYJ+rlu4W9YHXXS8NLcPuoX135fPeyE9ieuQRlcycQr5Lefd/7cyMfdWYmfN+2CHTU7Qpo1dUoarRTcymJuhVpDxojKMoZfhY52dnLwqqUlqviy4VwItHYChV9DGByamirZBNMl2lhSiG+TyilHzkm/zvM02y3XIgQRYdhFcbUE6UD3SJdEp8DO4fUBLa4pPFiRRG55JwfpCGtXvwEVCPRhpUjYkutSWYV3clOG+tDpkvX09PZXtR9t5LgW7ZHtj8rBcG7ejL8/ShW+DlEyyDdrkGCjXx+txuF2uQ+hG2ApjsCq1UZTDNZM6euB89rPzt8uosw4kJsOosaqsJReCSKhI9Vtd3AieJmsMRClEyWZLGPEX/8qVK6m5uZmmTp1Kn/3sZ2nt2n4/5ueff55KpRLNnTu3UnbGjBk0ceJEWrZs2bDXKxQKlEqlxD8DA4PRBTPvDQz2HIzoxT979my644476JFHHqFbb72VVq1aRR/+8Iepr6+PWltbKRAIUDKZFOc0NjZSa2vrsNdcvHgxJRKJyr+WlpZtehADA4MdAzPvDQz2LIyI6j/++OMr2wceeCDNnj2bJk2aRPfeey+Fw+EtnDk8LrvsMrrkkksqf6dSKWppaSE3YJHlt6gc06JsARWjihgJCWgrZws0pw1W9JrVpQsW0uMSnMgBo6bpltf+NFBsGNUPGJdiSctBDdR4tszPh7IBEVE4MEwOaaiDKujW+rC9lvtEY+VIAf/tQQS8Mloi63QxXFv5MVKiLGdBxDT0HhAR9TxJSaGVq6+bh2WnZtWvkmDxDIbNGJnLi0pzal+GabXQepAbMAKiblEfGPo+5bB81tKghN7vXg6iPepjpoDRuGDcYl54IpmYpKPANDZaHusJXpa+672hOcO8L+zMee/5iCw/kd8v6U8c1/k0rAnQGCiHERGpItPAQYjcZxcltY4JU3Ac5uuHX0cwOYyYJ2Uc+/IcpHAxylw5Kgv6+2DeQwRJHGpK89qAJYp8EPmSdIViIssf1maeWw48t0714xi3wbtCH/plUFhxbk1Ptle2W5PSw4mGWas9uexTOQqR+8ByvRyCdSijeUcM030YsVVpZVy4bwS8Ryxbe1fgECpD9Ejwoig2SunJBQXGdoen5G1YE9BTxQNZtxSVDVR691aedIYZ/h5bV2xoJJNJ2nvvvemtt96ipqYmKhaL1AOaDRFRW1vbkNrgAILBIFVVVYl/BgYGoxdm3hsY7N54Xy/+dDpNb7/9No0bN45mzZpFfr+fli5dWjm+YsUKWrt2Lc2ZM+d9V9TAwGB0wMx7A4PdGyOi+i+99FI66aSTaNKkSbRx40a64ooryHEc+sxnPkOJRILOPfdcuuSSS6impoaqqqrowgsvpDlz5ozYstfAwGD0wMx7A4M9CyN68a9fv54+85nPUGdnJ9XX19ORRx5JzzzzDNXX1xMR0Q033EC2bdP8+fOpUCjQvHnz6JZbbtmmisXW5sjnU2SXtEx7edZvg61c/TJIjbrWY2chW1Ub603OeKk3JVayoLK6obGy7XpMjEQ2y/p4IksX600FuHS5Q2rOUYh09+91HAmrsVXqdq3vcNamiXk8Bm5oBUnaoFsPanC2Jn/506B7g4tNpB0yHvZIdxQrx5pXaSK7reiZtMLt6PYH2lwvXk+6XIa6uO2Vw/2aysm2a2hH/ZPrjZES7YKskAuui/lGrlvD81wmX6tlMQOXwAC4kilNR/5n+8TKNmaNCwd5zBUSmmsS6HZWNWRsGyf1cg/dekCIFlpqlRwzR4b6Rb7UMFnPtgU7c967EUUqpGhcTEYzKwRYCjhg6obKdjYGfrOabGpHWMMuFSHrpG6WAZEUhXsuDCPMakckbUAQpWoer3rmN9TyPci2SXE98xuP11AHZOQD/TbWIOemDX6dbgz6XhsGlsvrBQ7lUgwzb8pzxge7K9vPYpbQoPZ8oOs7oMM/u3YyF9L6yIPsegcnuF/f9k+R5YZ5U5XiGAJRs7+BeeLD5iqjfZM8B+1sPFj3VZXU6xEWZHsMbeSKWp7sV1yDc3W8CARD0uak0MD1C7fD+g7ul6GOoV2otzZy34he/L/+9a+3eDwUCtHNN99MN99880gua2BgMIph5r2BwZ4Fk6THwMDAwMBgDGHUJulJTY2QEwhR1Wrpn4DRzJA2QhrFC0guD13PnCqORueGZbnMeKBSatkf6tKpj1a2r4udJesDVFG2CahZjPDmSfolO475t+nNnOSlMzJRlIuOx6AmzFtBLgtSQcnlKQfcW4At9fySGsKIesUE7y+3g3TRIC2tnV7uCyfLNNZgdzUeVsFuiB4I7nzYJ0RELiTRQPdC25L1zgNF5vn4WGo6t0PTX2VCDRfcNgOdfF+kJgdFH4T64fO5mptRdYjHyaae+sp2NsDPHfQNT7954OrpyACPZJf4vA2ZJJ8D7lZ2Vjb+b9P9lvS5TJmI1tHuBidrkeNatLEzIfbXgfvnK29x9Ma9IXJbKSIpU7KBlga3V0fmcBI0Mso6RaDqLc3FrRiDpC+gAoRbeTyEuuRgycD0dsAd0LdJlgvAPHNDXDl0ccuulnOzAAlynBoeSOWMnAvo/lsCmSiyCeajK8drjcM6WgGodeECSES+LJ9XqONGmdHAyble7x0vznEjXL+3Mjx/9ERBGH0zsk8v1+EpXs911zx8J+B2CRI7hTvk+plPcr8Kd+q8Fn0VZDgCqh7fT67mLj6QSIdIul+SX3sNQ/tjvZ0+bnvd1XQgoq2n9clwMF/8BgYGBgYGYwjmxW9gYGBgYDCGMGqpfn/GI1/RI1+3DEFml9jyuQBRknTKGyFyqweYWnKy0tTdAi6tBBTs39N7w/0llYIW2xipCwxhKdci6xYEy8/2DFP4kW5ZrqOTqfHmLqaagt2Y81t2oQeMXSkO19N+4mFyEw8spjFymX9jtziHcpDTvIk9DvQc9cFubpPYOrDwx4hZmvTgy4H1co6fL9Ulrf8bN3AdQh3cPoLWS2tjBpMxDRMB0dLoTYzqiNbLPi0yVhksf51qyBMP0eXifTptCUmM9uIxHOzSykGz+uAPTIaSa5TnNPl6iIgo42gZn3YTKLvf6l6XeLAPgnGIqFbgdtGTHFGJB4UT4u1yWNLfSLvmgELFSHvUriXSgXUAKWY3wvMxV6dFEoRoe0hdl6plX6F3kD/FdDNGi0MZiEh6oZRzEHWyR1sfQjAHAzgZeDOyWrbPvw9laSXxFq8bXftrUTVh7XHyPC9CDli3axQ+ehBgOUzG1n89LphOs+dENUYA1bq/CNeIQPRo9DTytMiBmBQn4EP9WJMC8Y8Ca0dYTy84/Hd1GT0iStp7CMcGeJm4cZBNe97fN7v54jcwMDAwMBhDMC9+AwMDAwODMYRRS/V7vv5/pXpJJxUTQFVBUA0rw4/i6sa9kNNa0DIFPTgQb8ej/AcmSNEYSGGJjffNNoPldVQL0AF04n9OXV7ZvnPcCaLcvtPYKjs1nrOX5cB7wI1rgUXAQhSpPBWW5dJT+ZkwORFSqm6NDFrhAIVfjkNu6ZT8/YhUmotJNIrgCTAoyQSX85q57etr0qJcvp4jI6HEgMFWvJgMhFOs4jYpjuOC5beZ0iwlpFyByYXydTQsJsW6KtvrQ8nKdm2S663cejyFINaKsBYv1GjUMFCnk+Odle01tTwWXM0boaT6n7WsmzjvJignPPJCHtVWyQA+FjTauGr2dnFjNZXtYI/m4eLymMf5jHIhkfTUsMHiHyUZL6hJJ9C+voK2KAxcSwuahVR/th4CvWjBlopV3PHBDvASgKAt/lk98uLLWa6L17J2kc5L638rwQ9obeY5jFJBbrysz8GRNZXtZ6sPrWzrCavQcwg9j7ryINdpw9LXx/VJFblfHC3x17DeXCD1OHnZR+F2nt8+LAcB3SxPjgX0+GjtYM+SRHaNKBfqxIRsILNAPTFIGhFREOh5EcDJk+2NCefEGIK1IthJAgMSw9YG8DFf/AYGBgYGBmMI5sVvYGBgYGAwhmBe/AYGBgYGBmMIo1bjH4AvLfXxUDu7cuRqWRsJdkJCBZ+m9flAW6ljjRhdb4ikq153J+vbBdDjctXyt5ILOhcmikEty8vL+/hyfI0CZGxBnY2IKF1kDc4HSXoCPawBuprLCLqbORlIyNEhI+WFN7N26E+xzlUGFybU6omIVJjrg8mTSlHN9SqE+hdveknW+vINmg4PkdC8Pm6TvpBM0uODCH+BXrBH0CLqIYJdPIaib/F9UZPUNUUvOHQERF23zblc10w3X9sPkcv0qrnwSBhRzi/NGURExXf62NAAI6SVtbZ/OdcfHi6fLxHRG7S7IdBpkxO0qadR2vbUlvk512xiV9K98xBdTenGPTxWcpBAKaRp08EevjYmdhERJLVEOmRxr6I9iA12QyHNPTPfxutAtI0HUm9Grg+oDbvgeoiuXekO6eYahjnX1w42SbLW5BXBfiDNR9G+CZPBEBG1lnkgojbty8jn8/fxsVIc1tIszwsrJ9chjLKad4d/HaHNgAvPgEmzSgm5FmIyrCK0Xama64NtSkRUBPfscAQEf1vWGxNlKbAbw6uVw/J5ShFYu9LwjgpI98kyJFlyMAkbXBzXhv5zTOQ+AwMDAwMDg2FgXvwGBgYGBgZjCKOW6ld+q59W0lwdMJmKDe4xxSRv+/skH+vrZZ87uwOi0VVr+c+BxgpEmb4pQ4g3f06jtcH9DalBfwoi+umuQNDsWYgWaGnFXHAZsqyh6fNBLkPIDEHTlUOy3kjV+yDQnXBX1NpeRJiCege7JF0W6oIIZUiDghtNoFcOPQV/OpB4BiMoEhHZQPnaRXhY4eIjaVk3xnUtQjKTqlXDXIuI7Bw/K1L92XHDU2nhBPOlmSzTzvE+zZUSKE3bhy6Jeh9xu06I9lS214XBnS8mrz0p0J8QJRvYPSP3lRIeuSGPAn5ZfxfmZkMdu/N5Aaa1PS0ZktfXV9kOQNKkYlLeMwC5sDxQC3A+q24p2OBcLYIramgjSmDyPhiNDutajmjzDB4DXY5RxgsltIxOsKZUN/ID9aakZNJYy8day+wKGcE88jXy2jODGyrb94Z5nmASIyI5t8q13N4zqnnN/XdaSnfK4euha+yb0QminC8NkTRBisD1zp+Si6EbZAo9ALKEv4cbshTXpESI4pjKQJ8n46KcsmGu+iBSYn7ouhHJaI/iWo4s6MCzoleuA+8Rn/R2JTWQkMq48xkYGBgYGBjoMC9+AwMDAwODMYRRS/UXqixyAhaVEpKKERHWykCJhCGiny3pDgsoa6+PTacxghMRkXKY+g8GmS7uhMhTOrWOSR2Q5sFyqqT9vsKEGBAqDPM1ExHV+LkOWFNhFa5F7ivG+V5lSNKjfJJm8iDffBEscCObuIyd1jLSgNeBjdbUGlOJFJeCJBgeJkvS2j4gkmUAxaZFoAu3w3n7gBU9s7pUTkoJB+UGjIaH9RT5tUlGUytUw34tmY+jh3J8F0mIPFeKyPpg/vYAJI/RpZ74mqETT2F0OTsjpZB3Cg1ERJTX5I7dBb4+m5yiTeFgcdgyvRluzxgkU/JpFs1OLVPZ8RDT15leUYxKcbD4BnrejWKSKz1pDG8jVWsXwSI/JMeKivCxbB1E2AzJRcUL8Hn5Op7sFshc+XY5ptDIO53VvBsArZt4MOtjp3L/nHwt/CU9g/9ASU3rIrTqR6+YVV3cD15aWrD7ell6WJPmcuFWWTdMuEMB8ObqAEk2JuUY7CNMXkaQLCzSpkVNhLFgY0Khnj5Rzi40VbYtsMrHNgn0yjmIUnIpymPGSkveHr2VUFLwQG5yilrEzoF33lZ+ypsvfgMDAwMDgzEE8+I3MDAwMDAYQzAvfgMDAwMDgzGEUavxB1OKHL8iX1bqJHaJ9RQXXbAC6Bali84oTIFbSERqQignRwJ836nxjsr2y/7x8hxoQdSPMdLUoIxUGd6xCUIwOZpm1gURr+KgU2M2Pd1VMNiLGecwG5TmKlZAdzWoIGjWyid1NivLohlG9fOkbCe0LIxkhu58Zd12I8YN6UGGK11CL0VBG4XqYWZE3T3GC4DdQxXXxwUd0skObxeCmf+UJov6wMDBcXg7k+exFdd+XqPeWMhx40W1YVsAdym8D7qF6T/d9wq1EhFRtrR7uvOpgCIvqEhpth2oaSZjrIl6AdasCwmtD8M8f7IF7g99vAYgch/qq0XIlqi0c4TdENinuJC10pfV7Eby3Fnhbuhszf6mFOYBEu3hOYPuaeEGLXuhB26NHujHuncXrJmoyaOdjr9KuvOhHVIgzeMq0yQHXzEJ9k7gVlYV5nUjTTLjZzkBdlU2G1/o8wznoN3N7VCsxsipujsfbwd7IZMr9F2+RovECsuSDfOZGmpEOQdfS6DxY/RN3dYsV8sPJWzAEtJVEF1KSzG4BtgX6fYVAxE8PWdomyMdI/7i37BhA33uc5+j2tpaCofDdMABB9Bzzz1XOa6Uom9961s0btw4CofDNHfuXFq5cuVIb2NgYDCKYOa9gcGegxG9+Lu7u+mII44gv99PDz/8ML322mv0/e9/n6qr+Vf39ddfTz/84Q/ptttuo2effZai0SjNmzeP8vn8Fq5sYGAwWmHmvYHBnoURUf3f/e53qaWlhW6//fbKvilTplS2lVJ044030v/8z//QKaecQkREv/jFL6ixsZEeeOABOvPMM7f6XgPufLkmLdJTA9BQWYiABq4yxYTk5dClzBeYVNl2w5JPKtQyTVILrnT7RzdWtl8IHCTO8aAFLaB/MDqYPyJ5mTxQZH/ZMK2yHUhJmiYLskYMngFd8ywtolgZvHzK4OoXrM2JcrlxXDDXzOXC7Vy37HRJb0VWs+tNsQES7tRLjjoJH3qlONBg3VwHNySHXno8Jt7gvjxy2tui3DvhfSvbhQaut78X3LpS8mWTbwIfLYiUhy6Ng7KZAEeKEozuuvhqF7v1pNvxPtxHYc2tC+nAbJ7rgG5lRHJsPbORx22kFdy9ZBfRfoF+qj8dGNoVcFuwM+e95yMiH1E4oEVfhE+UQpHHVKITkk3VyTHldXTyMR9HsNNyT1EgPbS7Z2w1/9G3r5zDKB9FN0Jk0Nzwbq7+HqDgXYjCprnzBfrQPZDLIQVsaRoYujJiIh5KyTYJNHJ72euZdncKkKgoLdeUAuiZKOPpLmVYJaSiUzlYw7UEajhXu/O8JgV6RDEqCNllaNdty5XXDneA+zG4QlKR+8jfp7UjRO4reJhpTV67ALKGt6kN9rMUbJXlOUjvC+lBc+fD6LShDmhvkE90KSS0uf+YuyMi9/3+97+nQw45hD71qU9RQ0MDHXzwwfTTn/60cnzVqlXU2tpKc+fOrexLJBI0e/ZsWrZs2ZDXLBQKlEqlxD8DA4PRAzPvDQz2LIzoxf/OO+/QrbfeStOnT6dHH32Uzj//fPrKV75Cd955JxERtbb2f200NjaK8xobGyvHdCxevJgSiUTlX0tLy5DlDAwMdg3MvDcw2LMwIqrf8zw65JBD6NprryUiooMPPpheffVVuu2222jBggXbVIHLLruMLrnkksrfqVSKWlpayJ8mcgJEwU4tAlkn811+oK1KGUjI0Dt81DKni3lWv2Z1qWygEANMQa0vspapUywuRNmCvDWCzvWHtPr0MCXcMIMLtkVqRbHGKo4WVVaYKBzuWSVpQgVeCxhly3Plbzyk9tBiuQy5ypHaJyKyUsyD+QKYmEJGCpMW9kBPAX0+yPMCYEf5md7urRPHHKDsLIzciKxcWEo9hWqua1UdWw6XItyvSK8REbkRvkYeJKBStaz3KePerGz/3Te1sr2+na8d6ZBt3zeB6xOq5nHmaBHTSvX8UB8ev6qyvcw+uLLtSVaW9vL3XyPl335U/86c98ruj6g4LdEhyr8WaYSyTAnXJLbw7eIHj4kAc88FjerHaG1IZWPSGX9Ukx6CTF/n67kTIuBNUdYknuI4rkOmieumNko5Ez1wkMrGMZ5FWYmIYtGh28HyZB1QIsiN53kW6gQPl4hcUw6OrKlsPxr9MFxb3gtpaYy0F/Lz9TIFzVMow+PfBXeXfL0WaRTGuVPL0ooD1xvkzYPeMwlIpOTAs2qJc8qQdE1Y9W/aLMrZZV6XrBAveOUYJv6S7RjqQo8i6Be/nPcYMTLYzeV6Z4BnV1BrR1f+/14Y0Rf/uHHjaObMmWLfvvvuS2vXriUioqamfr2zra1NlGlra6sc0xEMBqmqqkr8MzAwGD0w897AYM/CiF78RxxxBK1YsULse/PNN2nSpH7DoylTplBTUxMtXbq0cjyVStGzzz5Lc+bM2Q7VNTAw2Nkw897AYM/CiKj+iy++mD70oQ/RtddeS2eccQYtX76cfvKTn9BPfvITIurPGX/RRRfR1VdfTdOnT6cpU6bQ5ZdfTs3NzXTqqaeOqGJ2WZFjKZFghUgmwinlofoiN7uWSQfp5jDTMm5UUsJIq7Rl2eJ1WqydC2lGk0h3oSUrWmQXtaQZmKQlW2IOK9wlubNUninAKJgiB7u4EtmkrE+gl8sV0mAxbss6BCAntb+LacNwB9ShpLcjWBsDRUeW/FobLikO+fh8Jy+vHdvAjZIC6rNNS47SkgcarB0aGbrf6ZMBSEId3M+b4Fnrsa1qZMd6Aa44BnXx90mK7Y0DmYLe2MXBmIIg75SDksr1gxV5TwfT1iEtnzwmHnqzt6Gy7YCDhh7I45epfq08ly4T0TraHtiZ8z7QY5ETtOjfHZIp8EGXOmAZ7uR429Ii81hBnlsbOqFvtrDqofU3rjXF0tAJbYi0nOndbKFtKRmsBteo2EYe790HaP2egQXC0Racyk3ln/4sWLqDp4iKSu7XXcV1CkKbosW5tVYmAPrTtA9UtkM9QFcHNIoaJT44hDJLh+aNoKI8N9IQ4AiDCxERlSCRWBnWB8vja2sxn0SSJEHp4zqmLXEuJEprSsIEDMr1U0iLOVhLMShSQbZ9IYnSJMhLrVJGCHSyZ4BYP3FYaHrVwLMOkzNsEEb04j/00EPp/vvvp8suu4yuuuoqmjJlCt1444302c9+tlLm61//OmUyGfrSl75EPT09dOSRR9IjjzxCoVBoC1c2MDAYrTDz3sBgz8KIQ/Z+4hOfoE984hPDHrcsi6666iq66qqr3lfFDAwMRg/MvDcw2HNgkvQYGBgYGBiMIYzaJD3K6dfJMfLbIIBmZoHoovzy94xdBN26gKKodIlBXb6jmxMnBJsgoU1OczOBc0ogdaNbRTkrm9lKcn1m16+ubC8rN4hyYYgeWB7GXQcTNxBJNxE3ApXQxJ98PVOwGBHKgbYq18vkEb4eiLwXZf1U19ZQf0IbCOnaJ8/JV7OYVa7l526q7hPllMOh6tB1RtheaLYJTh7aweXxhHXTXZOwri64EgVkdagjx5ppCRLuREKYXEU+bBnZb3S5HF5Gpq4MRJ6DcVaslTritEC/Xpjx755JeopJRXZIkc+VjREA7d0F11RfijV1NxAR51CZx0E8yjpsuSi1d9SCfRkYu9CESptnwrYH9GMvxlqwrsPasF6hy55VHD7amq+XhXjPB9p7SPavciBCZhLmaatsE4zm6YZgTVmD64acDEckOBTnSsXeHa6m4uAz2ZikJwjJvcqaCzXYCdTFeHK1hROiXBHWTKsa5hZEAB10bbT7AVuQcjW3iZ7YyQ+JlTxcpEDHJyJyoM8s0OsHkuUQEXkR+e7CZE4hjNwXljYVFow1jMxp1XAd8nXynAFbBXf4oSTLb10xAwMDAwMDgz0Bo+6LX72bQtct9f+6KWvpRT2w5Pdy/Iseg7mUy1oqX4yZ7BagnPwVB4fIgxS0hTRfzy1q58AvPw8N3aHaXk7Wx8rx1wxeu1yS17Yz8GsfjrkQtALboP8Ytg9XQmlxo8sl+IUPbYrtXS5L63hsIBeuh33Sfw30DABrXDhfT7vqFrHe0N4ZWQdsI3FfuBzeh4ioXOZf3tgmeE83r32Vw7O78OWmXZrK2EfwVeCCyfSgMQPVFvUpaNMRq5TF+kAbaP2f6eu/Xjbd3wdKbaWZ7y7GQD29d5P6uFnZ0C72OxzDvtbbuazgy3CY9iMiUsj8FLGvYW7n9HEIeUKGGeN6fTyIp19GlmDQ2OM+ddzh+l2vD8T0z+Ic0YLaQDx7qwQpynEdy8m1IgfpbnH+4fwhInLxcWF+lzLcD5725VyGk3Au6X2E88QKQnpv6Ac9Vj/O23KZz7GgfV2dbQFCGOuDY0mvHx7zICnVoPdLcZh1Vr92Htd63o/9qsfkH3BkG6jXe817S42ylWH9+vUmfKeBwXbCunXraMKECbu6Gu8JM+8NDLYf3mvej7oXv+d5tHHjRlJK0cSJE2ndunVjOqrXQCjTsdwOpg36MZJ2UEpRX18fNTc3k22PfkXPzHuGGe/9MO0w8jbY2nk/6qh+27ZpwoQJlWxdJpxnP0w7mDYYwNa2QyKReM8yowVm3g+GaYN+mHYYWRtszbwf/Z8CBgYGBgYGBtsN5sVvYGBgYGAwhjBqX/zBYJCuuOIKCmoxkscaTDuYNhjAWGiHsfCM7wXTBv0w7bDj2mDUGfcZGBgYGBgY7DiM2i9+AwMDAwMDg+0P8+I3MDAwMDAYQzAvfgMDAwMDgzEE8+I3MDAwMDAYQzAvfgMDAwMDgzGEUfniv/nmm2ny5MkUCoVo9uzZtHz58l1dpR2KxYsX06GHHkrxeJwaGhro1FNPpRUrVogy+XyeFi1aRLW1tRSLxWj+/PnU1ta2i2q843HdddeRZVl00UUXVfaNlTbYsGEDfe5zn6Pa2loKh8N0wAEH0HPPPVc5rpSib33rWzRu3DgKh8M0d+5cWrly5RauuHtgLM17M+eHhpn3O2neq1GGX//61yoQCKj/+7//U//+97/VF7/4RZVMJlVbW9uurtoOw7x589Ttt9+uXn31VfXSSy+pE044QU2cOFGl0+lKmfPOO0+1tLSopUuXqueee04dfvjh6kMf+tAurPWOw/Lly9XkyZPVgQceqL761a9W9o+FNujq6lKTJk1SZ599tnr22WfVO++8ox599FH11ltvVcpcd911KpFIqAceeEC9/PLL6uSTT1ZTpkxRuVxuF9b8/WGszXsz5wfDzPudN+9H3Yv/sMMOU4sWLar87bquam5uVosXL96Ftdq52Lx5syIi9fTTTyullOrp6VF+v18tWbKkUub1119XRKSWLVu2q6q5Q9DX16emT5+uHn/8cXX00UdXFoCx0gbf+MY31JFHHjnscc/zVFNTk/re975X2dfT06OCwaD61a9+tTOquEMw1uf9WJ7zSpl5v7Pn/aii+ovFIj3//PM0d+7cyj7btmnu3Lm0bNmyXViznYve3l4iIqqpqSEioueff55KpZJolxkzZtDEiRP3uHZZtGgRnXjiieJZicZOG/z+97+nQw45hD71qU9RQ0MDHXzwwfTTn/60cnzVqlXU2toq2iGRSNDs2bN323Yw835sz3kiM+939rwfVS/+jo4Ocl2XGhsbxf7GxkZqbW3dRbXaufA8jy666CI64ogjaP/99yciotbWVgoEApRMJkXZPa1dfv3rX9MLL7xAixcvHnRsrLTBO++8Q7feeitNnz6dHn30UTr//PPpK1/5Ct15551ERJVn3ZPmyFif92N5zhOZeU+08+f9qEvLO9axaNEievXVV+lvf/vbrq7KTsW6devoq1/9Kj3++OMUCoV2dXV2GTzPo0MOOYSuvfZaIiI6+OCD6dVXX6XbbruNFixYsItrZ7AjMFbnPJGZ9wPY2fN+VH3x19XVkeM4gyw229raqKmpaRfVaufhggsuoD/+8Y/05JNP0oQJEyr7m5qaqFgsUk9Pjyi/J7XL888/T5s3b6YPfvCD5PP5yOfz0dNPP00//OEPyefzUWNj4x7fBkRE48aNo5kzZ4p9++67L61du5aIqPKse9IcGcvzfizPeSIz7wews+f9qHrxBwIBmjVrFi1durSyz/M8Wrp0Kc2ZM2cX1mzHQilFF1xwAd1///305z//maZMmSKOz5o1i/x+v2iXFStW0Nq1a/eYdvnoRz9Kr7zyCr300kuVf4cccgh99rOfrWzv6W1ARHTEEUcMcut68803adKkSURENGXKFGpqahLtkEql6Nlnn91t22Esznsz5/th5n0/dvq83wYDxB2KX//61yoYDKo77rhDvfbaa+pLX/qSSiaTqrW1dVdXbYfh/PPPV4lEQj311FNq06ZNlX/ZbLZS5rzzzlMTJ05Uf/7zn9Vzzz2n5syZo+bMmbMLa73jgda9So2NNli+fLny+XzqmmuuUStXrlR33323ikQi6pe//GWlzHXXXaeSyaR68MEH1b/+9S91yimn7BHufGNp3ps5PzzMvN/x837UvfiVUuqmm25SEydOVIFAQB122GHqmWee2dVV2qEgoiH/3X777ZUyuVxOffnLX1bV1dUqEomo0047TW3atElc54orrlBEpNrb23fyEwyPBQsWqEmTJm3TufoCsDVtsCfgD3/4g9p///1VMBhUM2bMUD/5yU/Ecc/z1OWXX64aGxtVMBhUH/3oR9WKFSt2UW23H8bSvDdzfniYeb/j5/2ofPEbbBt210XAdV11++23q5NOOklNmDBBRSIRtd9++6nvfOc7u/VXrIHBjsbuOueVUuonP/mJOuqoo1RDQ4MKBAJq8uTJ6uyzz1arVq3a4XUc6zBW/Qa7HNlslhYuXEiHH344nXfeedTQ0EDLli2jK664gpYuXUp//vOfybKsXV1NAwOD7YgXX3yRpkyZQieffDJVV1fTqlWr6Kc//Sn98Y9/pJdffpmam5t3dRX3WJgXv8EuRyAQoL///e/0oQ99qLLvi1/8Ik2ePLny8tcDexgYGOzeuOWWWwbtO/XUU+mQQw6hX/ziF/T//X//3y6o1djAqLLqN9g+6OnpobPPPpuSySQlEglauHAhZbPZQeV++ctf0qxZsygcDlNNTQ2deeaZtG7dOlHmr3/9K33qU5+iiRMnUjAYpJaWFrr44ospl8sNut4DDzxA+++/P4VCIdp///3p/vvv36r6BgIB8dIfwGmnnUZERK+//vpWXcfAYKxid5vzw2Hy5MmV5zHYcTBf/HsgzjjjDJoyZQotXryYXnjhBfrZz35GDQ0N9N3vfrdS5pprrqHLL7+czjjjDPrCF75A7e3tdNNNN9FRRx1FL774YiVS1pIlSyibzdL5559PtbW1tHz5crrpppto/fr1tGTJksr1HnvsMZo/fz7NnDmTFi9eTJ2dnbRw4ULhmzxSDESkqqur2+ZrGBiMBezOc76zs5Nc16W1a9fSVVddRUT9bn4GOxC72sjAYPthwNDnnHPOEftPO+00VVtbW/l79erVynEcdc0114hyr7zySsWlZADoXjSAxYsXK8uy1Jo1ayr7DjroIDVu3DjV09NT2ffYY48pItpmC9+5c+eqqqoq1d3dvU3nGxjs6dgT5nwwGKx4NdTW1qof/vCHW32uwbbBUP17IM477zzx94c//GHq7OykVCpFRES/+93vyPM8OuOMM6ijo6Pyr6mpiaZPn05PPvlk5dxwOFzZ/v/be/cguc7yTPw9lz6n791z0cxIlmQLI5CxYzA2NsLOhoBYF0tYCK4sSZHCZF2hYGWCcbGhXD8gC0WQN1tZCFljLxRrSC1eJ94EgmHBRUQwxa4v2GCwAwjfJVuekUZz6Zm+n8vvj/H0+7zvdAuNrctI8z5VKn3T55zvfOe7ne7nvTz1ep2mp6fpda97HaVpSj/5yU+IiOi5556jhx56iK6++mqqVCq989/0pjetyEZ1rPj0pz9N//RP/0Q33njjijzdBoNB4nRe89/+9rfp//yf/0N/+Zd/SVu3bqV6vb7q5zesDkb1n4HYunWr+HtoaIiIiGZnZ6lcLtOjjz5KaZrS9u3b+16fyWR65f3799PHP/5x+sY3vkGzs7PivGVFsaeffpqIqG99L3/5y+nHP/7xqtr/t3/7t/TRj36UrrnmGnr/+9+/qmsNhvWI03nN//Zv/zYREb35zW+mt73tbXTBBRdQsVika6+99pjrMKwO9uI/A+F5Xt/P0zQloqV0qI7j0Le//e2+5xaLRSIiiuOY3vSmN9HMzAx95CMfoR07dlChUKBnn32W3vOe91CSJMe97d/97nfp3e9+N73lLW+hW2655bjXbzCciTid1zzi3HPPpYsuuoi++tWv2ov/BMJe/OsQ5557LqVpStu2baOXvexlA897+OGH6Ve/+hV95StfoXe/+929z7/73e+K85bzST/66KMr6tD5p4+G++67j373d3+XLrnkEvq7v/s78n2bngbD8cBaXfP90Gw2qd1uv6g6DEeH2fjXId7xjneQ53n0iU98oveLYBlpmtKRI0eIiH9F4DlpmtJf/dVfiWs2btxIr3rVq+grX/lKjwokWtosfv7znx9Tm37xi1/QW97yFjrnnHPom9/8prAzGgyGF4e1tuajKFphRiAiuv/+++nhhx+mSy655NgfzrBq2E+qdYhzzz2XPvWpT9ENN9xATz31FL397W+nUqlETz75JH3ta1+j9773vfThD3+YduzYQeeeey59+MMfpmeffZbK5TL9/d//fd8Fu2fPHnrLW95CV1xxBf37f//vaWZmhv76r/+azj//fFpcXDxqexYWFujKK6+k2dlZ+o//8T/St771rRXtPZOUuAyGk421tuYXFxdpy5Yt9M53vpPOP/98KhQK9PDDD9Ott95KlUqFPvaxj52orjAQWTjfmYRBebtvvfXWlIhW5MD++7//+/SKK65IC4VCWigU0h07dqS7d+8Wwg8///nP0127dqXFYjEdHR1N//iP/zj96U9/ukJQZLm+8847Lw3DMH3FK16R/sM//MMx5e1+8sknB4qWEFF69dVXv4heMRjOXJyua77dbqcf/OAH0wsvvDAtl8tpJpNJzz777PSaa66xXP0nAU6aKt7HYDAYDAbDGQuz8RsMBoPBsI5gL36DwWAwGNYR7MVvMBgMBsM6gr34DQaDwWBYRzhhL/6bbrqJzjnnHMpms3TZZZfR/ffff6JuZTAY1gBszRsMpwdOiFf/3/7t39K73/1uuuWWW+iyyy6jz372s3THHXfQvn37aGxs7KjXJklCBw8epFKpRI7jHO+mGQzrAmma0sLCAm3atIlc98QTey9mzRPZujcYjgeOed2fiBjBSy+9NN29e3fv7ziO002bNqV79uz5tdceOHDgqDHd9s/+2b9j/3fgwIETscRX4MWs+TS1dW//7N/x/Pfr1v1xz9zX6XTowQcfpBtuuKH3meu6tGvXLrrnnntWnN9ut0Ve5vR5AmLbhz5Obpglv6UuSLmYveJIr7zw8HCv7NXlL4azXn+gV57++uZeOVhMxXlH3sTt+Oxr/lev/In//EeyzW/lFJXO96u9cuWJbq88f24GL6HK43ws+/1HeuXFt1wozis+zZKUj/+7Ij/DP8fivKDG9TndGK4p9MrbvzIvrjn8Wm7r8CONXnnmgrw4L5zjfqn8fLZXnn3lEJ8zI9tT38hTafhfOGtXc6NMvZud7vTKUY7FQp77zUCct+Uubt/COVxHcT9PiPqmrLimMMXjt7CFjw3/6LA4b+6i0V556nIWHdn295E4r3ZO2CtXnuT7zm6H+6ofpx6kGB/+yVyv3Nkg+9iN+b7z27i+dlVWWH2M29Qt8jf48mPcxwd/qyyuKR5MKO626KGvfYpKpRKdaKx2zRMNXvc7rv44eUGWxn68IM4/+Jv8HNlpnp+j3/xlrxztOFtcM38uz5vGOPfrWXtr4rzay3md5Q7xulrcxHMyq+Z78eGDvfJj79vSK599J8/bxS1y7ofzPJbhYZ5PSVaK5vhHeA+Ih3jeLG6G+hQrUn2A21N75Ua+5ixZt9fmvsM+ifP8efkxcQlt+OFUr3xk53ivPPMbcv9MoUlei//wX8Jj2X1azseX/AMfm7qU53G3KE6j4V9w373+/+M59U+fubxXbpdln+SmeY0V/vGBXvmJL/Ce6/lSdOj8s57rlX+2/6xeedsX5XnP/hbvs2ff+nivnJy1oVc+8ir5rFGW27dx76Fe2VlsiPOO/DbP4/KTzV7Z7XAfNMfkftItL+0Nx7ruj/uLf3p6muI4pvHxcfH5+Pg4/fKXv1xx/p49e+gTn/jEis/dMEteNkteqg7A316eN2Y3y5unF8kJ4Bf4PC+A8wJZuZvn6wolr+81S/flReuEfMzPwDWhfPHjMd/JwOeybt/jwcVn8jNy4/F9rs9J+Zibg2s8+a0Jn8OHCa+fz89wv/he/77T7fECnkq+H8F5qm4f6Cd4BjcbqPP6tw91e1bWzeMnroFnICLy4Do3x/fBdi/VEcIx/PwoL36YUnjfxJdtdZ3+z+eFau5muE1JhvvO97pwjZqfGa77ZNDmq13zRIPXvRdkyQuy4vmI5DPiuvUdmDeqjwf1q+9JARicD7iuvIDrXrH+3P57D87blfMT1gVMlESJUeEe4Pi45nDeqXkC7cHzvFC9+MGyi32SZuFzuRQH7gFuVr34YWm7sDC8PI9lnNX7Xf95nMglK9ZBWOT9E8fOC/TagbGAPdfNwzOoF3+mEPQ9z1fnYVt9l69JBvQVEVEq5iCf57h638H7ch+78eB9FfcGol+/7k95rv4bbriBrr/++t7ftVqNtmzZMvB8F/aDuRp/6/G7/KD5KTkhF7tqJj8PR65lSrrceffVX9orhwty0I88WemVR+HXcafMiyw7I69xYz7P8fg+ug1uC178HZgoTbXxTPGv+aTK3z7TgpxECLEw4dujK/dYyh6BD7ow2VqwOXTk88XwbTYJeVp5TbVg5vHLCE/ewjOyDfirJxiDTa3G1wcl9eVqnjd0N1I7B8CFrvQWYKNvyS9KfhM2fqjb68CiU19Mwxo/rzvLvyyTzfIbePAMP5+3GTaBRC7YFPZtrwM3g4Udy25gwm8NY9C6j3JEaUjktNWkhGmE85gSftD2sFznCfRLAjsdMmRERJ0i92XxWb5RpgmbblfJ0QZceQJfRFL4Yhvl5Fhm6nwshRdyt6y+9B6G+lyuI8rx9WFNbxywp8DLPVbLAOdXpwpzFfbPVKv2wlzrwK/qNCP7JM1zmyL4xX7uMLOG+2YlCxIXuB/xV776zUJuxM90WYF/YX9v8YpeeWGrfJ15He6TYgh7SMDt7DTk4vnVEf7Fjl/2Ul92SukA7OdwLMUXsHr3dnELwBfzUdzsogI/U9Dh9rQr8kWfekv1JcmxfdE/7i/+0dFR8jyPpqamxOdTU1M0MTGx4vwwDCkMB2/SBoNhbWO1a57I1r3BcCpx3N19gyCgiy++mPbu3dv7LEkS2rt3rymsGQxnIGzNGwynF04I1X/99dfT1VdfTZdccgldeuml9NnPfpbq9Tr90R/90a+/2GAwnHawNW8wnD44IS/+d77znXT48GH6+Mc/TpOTk/SqV72KvvOd76xw/jkqnKV/2gaOfzsu25hSF2ztJeXxmmF7YQMcwJxE2laKVfaufHX+qV7526pp7jgboFKXbVYB2N3mXiptR8EikytZcBqKg8E2GQebdxSbbRKAo18bnf608w3YC0tMs2pbZLfIdQQ5bmsEzikZf7AtGhFnJanUHWW/jDjkY92irC8pcvtaVa68ALax5gY5fbOHub7Eg3GO5CRCm3gS8hyK8nLM0JbcGeFxbg1z3eGs7GP03E1G2EvZ1T4R2P/gVJqq6dApcCMyDbArRlxfKIM3lnwxuifXyH9c1jwR+e3nHSSVSV2sBRiXpMlr0WurPkaHU3Cc1usiU4d+1bb85c9j1Z9YBxRd8B9AD/qlNvAx9DPwF5U/g7hx//3KVeObonMsHApqut1QdQx+IkV+7kTZsx3w80HfpYVKR5yXRrhguI6XlqZ75V91NuMllIC/E7YtVVw0+jdkHegv7Hvl3iTGFfqx2+R1HhbkM+Dy8zx+1lg5a7dG4ExwzkzhebSfQgzOkGmGr3FC6eOBdWdnuD63wW1NMjLsYdmXIx6wD2ucMOe+a6+9lq699toTVb3BYFhjsDVvMJwesFz9BoPBYDCsI5zycL5B8JtEXrKSLokgb0E2C/GhkDCi9Iykdh8/wClDyxjSFkkabGG6QP2gaad4BmJ4oQ6ktfOHJWUYAa1NQPP4LXlekoMYf0hEtIJqBHrIhTAPgiKGFuk6VtQHQJOA08Y4cggvbCn6PMQyUO7KJJCZhYQUQOe7KuQSwxqR4kbqM1hUfVyCeFqY2UlZJc8ZEPGI7SbiEBkiIg+eF8OOsE+IiEYfhhjsBsSLO3JueU2eu8I0pfMCQAhfawhoxDbXF6to1VbVpbhzmn6nT5b+pVm5NaEpSYTmAbWqxw9p7tpL0JyizVQQrlbhzgznYCxX5BPhD7KHgI5dZDrWjWToWnuIGx4+x+fNv0yGeg4/zdS4V+fz/CauF9WgSb7GOZcTbaFZikgm58od4mP1c+AkbX2E/sI5mDTU6yODJhMwh2G8cEnlZ8B9xBkc5RHDOqunYCoF82NXdqNYO06OxyJtwzNk5cPWG9yGQp7Xrx5/NDmnC5yEyJ/nvaZbkPuO1+R7ufOcgCudlwmlskfYHBLOyJwTyyg+I00U7eGl90Z0jCa+03R3MBgMBoPB8EJgL36DwWAwGNYR1izVH2WXMnj5i+oAMDNZSOO4CFSgzmpUHYZKgFLW9E1QYvpkk8+u0p2irM8bZro6yjOFFM4z9dzYoGhH1AWIIcOV8npHer5T4WuaY5LPRQ/fLmSwG9/OlF+Sk/Rycwwo/J9iRIQ4TUY7oJcqUmfKSx1NMpCNdoU5Jc7zc3SLPP1aG5R3fBnSVkL2P6RldVQGemQLD/iGosuQ3QM3+kRFWAhzCNwrAC96bdJpjAGdO8Pco45uaI/yvMEMsomi7QPIGtnNQ+Y2oACTjPQ4diOidHACxzWNTpnIC2lFSlod7bAMJwtZD5X5SjDtaC3S0S6YYx4iA2LwJPfVfE8hc1+c60+v4lwlIvJgfsYVblx+StLfaRFoacyaV+L2BAsq3Gmk2ivi8/lNeRpS43rNHQswe2d2WGW69LlNi222h/3LPGsH6Ogk3LsGmeCIiDygsL81+yo+AF0ccILAFYhn4SAMeNSWr0A3MyCqIxk8Z8iBtQ0aHNk5eU17CMxNOTBrtAabOEQmQBjXblm2e9nc17/1K2G/+A0Gg8FgWEewF7/BYDAYDOsIa5bqX0YkHSOFoEyzwzRRaxQSmszI7zOlkCl8yIEgkqMQEWVDrnwYeCchjkJEURtdjLmIVKPwRFdIO6BIpbwwMUmD12GqPpiP1Hkg87vItO9T+1meuNqUdpIM/ImJRvymosyRRYS24nmJUv3CccrUQG7VlTSWP8/cI1Kh/qLkuDMHZ3rlxlmbuG3Qbp3cyatzW1OPTQVOR1KpmIgITRHhEekp60Y8vxyg8FD8Bb2ciYh8GHdvlrPGBN7g79jChKKeCROX4H2xH7ollSimlRANSESz1hEsLJk+tEgPeqML+hpo1kxNXpNp8BzN1AfYCojIh0Q7XhPWmYOJWZQZCJLahLNw7ChRQ+EsrHuYG1G+Is7DZ3eB6keafUVUzhx7hsfBGA1CBkyO2Wmuuw3Jd3S7UxE5AaaHlnx9tOpgvmrwNVMLbPJKW0qCGKJ3MFojmJPtRnPfNES0+HVQ4GsqkyxETDmQZMevgahOXV4Tl3hd1epgQm3JvSGzOMC8gwJlKoETmggxqZiuSYwtluEaV72TnOfHxTnGZW+/+A0Gg8FgWEewF7/BYDAYDOsI9uI3GAwGg2EdYc3a+J146V8iI5XE34WAbWHt+uAwLxfi9vD6QNlpEojR2Oix0TpR9j0M+cD60A64ImMd2KIc0CFvl+V3rwKIQaCoQxLI86IK1+GDbdvJgu3I1Zn7uBxDZrS2yu7lRmwDyxfYVt4chXCimg5943KnCgI0BWnTSyCcD0NVdChPmuf7YjhlBGFQkcq6Jb7GClEX2Q8hhMjFJSjnlc0SxIGC2f5t1aGQWDeB2EmsfCIyC2wzlKFBNBBo442LEJYay4sS36F0UPzbGkd2LiEvk1Cakf2FGegKB/uHxmqgABb6XjhNGd4p5hFkrWxXeT4U9zdIIOFxbg/1F0/SNt7mKI+ZV+f57bXk5I9G2SaOIcc4pN2S7J9wuMp/oH6M8jXC0NFOCX12cI9U+wb6W8D0TupqcwahNBcy9xXAx2peZZTEcEX034pUEtUOCIe9pvpUr/xN/1waCLyVx9djBr1OVdvKYf9EfwRXrqfGJnjfQEh3VOJx1Vk9cVyESI/y/0E/CiHSBL5KK0KZl/88xghN+8VvMBgMBsM6gr34DQaDwWBYR1izVL/fWtLl1jrvLlBVcQLUOlBVbqRoaODLhPb5vMqs1GLq6r/Obu+VQ5Uly3X78ymC2tGiDpgEDkQdSgeUHjTQakcT6fFBDCSBEBL3EAh51KVQe+pz2FDmMMf2VZ6QIXfZIxBONMNtzR9iCtJtyz5BCjE8zLRo6knOzumA8EmC4XIkgeFSkBERwwGzweDvrcLU0pVUKgp+ID3pNWUjUheyioEQUm4azAMq258Y9zaP0dEyf2Hopw4PxNAqDGPzGhBuqhITpu7gTHdrHY0NLnmhS9VH5Fj4kCSuMQFrGKh+baoR5qcyXKO01dGEguMsBLiUqQb/cqL+cw0z7RERFUCYx63zoEVFqS6D4bBxmddmY4zrKx9QGwyYHnDsdXgXZvUToluLEBapTKCYSTA7NzhezCvws3d96DtoUJpVwlo57kkMSxUhkiTFzLIO7CFx/72diKixgedDFtZf52zuXzeQ82y0yvvi4Rk5LojsYa4vbfHkxDBbLZ6FQmYY9kctmQER33mY3dQvsvm5XVUvxuXmHOO6t1/8BoPBYDCsI9iL32AwGAyGdYQ1S/Uve/U7yts7AbokAFEI1H9O1FNtyDF9Mwv16QxVuRxTcburv+iV/774JnFeBBmrPGTqj9E7G7HCO/cAem6ix6ui9oCyQ83uBMQfUEiESArpIJrDyosXKPgAaFGkDbUnMkY3JDkU3lDepy1+vsQHTXmtUxGCmA9QeBgVgF7XRET+AggKQX9pahepNG+e60gysq0u0IhoTkFTgc7QiJ7cTh2yFLrD4jwUFEKvct0PaLJoVfn58pOQfUzNtSjrUOyenly/21n6NaK9+oWQDgpCZQZvYRhxgXtCUpCdnMCtMDInDiFjXVa1B+Ynzpt4pAgnyfY4sBY645DN7ihZAR1QbQrBNLlCWCvqH91Q3yTrrjwB2UUXYW5NwF76mNoPGtzh7QqYH9vKG51gr4D9OE4gGqgs7VJeG819sM4Vk41miemI+9hvYCZPuc7D2gCzRB0yMvrynNF8vVc+9MQIn+fIdguBNxiv9CgZOtH05DThxaHMgNlZFBvDFxYXUbyLiDN8JoODXATsF7/BYDAYDOsI9uI3GAwGg2EdwV78BoPBYDCsI6xZG3+3sGTPDxbk55jdqQXqfD5kY4qzJPDYzGjfe2hlPM8FhT9Q5tKZ+/IVtt12ymyDyx0BW58K5UjB5urmISugyvAXVyFkYwQU2MrS6OV24QZg708DyCqYV5m14FZJmUN0dBvQNplUoD2QZTB7WCthwW3QNq7qTvNsX8UQN22zRBuvyJSHmayOEjKJ9R0tlA7t/To7IgJtvMKfQQrCUQLhfWkB1MpU+CPBnBIZJNUzBTUIkSrgjcFWWycBNyJKlW/M6QI3Ssl10xW20i6YzqVfDWRU7Ei7J/qk4D6CIaVE0seiC+sZ1/CKuQY29e4w38htYTifXlcQmrnIEyfOqXUKGR9x38B5p31+ojEO1RUZC5viNFFfnIMwxvZR/ExKKo3ecl1ZOaedqP/6wSyS3amcOOakoFgIe4ir1hX6zixG4McEz6MzJeJaSiNQ7oTBLBSk7X7/XJXPg720PSI39A76OhR4j+yWeZD0e6PDVVM8BAqDsZy3qBrrgz+Ct8jzoj6hwkuf77vYOTbfnlX/4v/BD35Ab33rW2nTpk3kOA59/etfF8fTNKWPf/zjtHHjRsrlcrRr1y569NFHV3sbg8GwRmBr3mA4s7DqF3+9XqdXvvKVdNNNN/U9/hd/8Rf0uc99jm655Ra67777qFAo0JVXXkmt1gCXcoPBsKZha95gOLOwaqr/zW9+M735zW/ueyxNU/rsZz9LH/3oR+ltb3sbERH9zd/8DY2Pj9PXv/51+v3f//1jvo+TLv3ToUoY3odUf7cINFpDXjRRYC70cEeGVSFmITPdnY1yr+yqrHmNGtsSRhYw6xeEcinWCcPa0nQwtesg7QPhfLofBtHSwxs5W5/bUoIY8CdmOdOCGCJb4jFSR+0hKI+AwI42UURA9cGz6wxjqc+NxXZj6JM2DyCVivRtmpPcJZoOkuzgTGQr6l+uG8ILNSXZHeVj+WHmp9tDkirM1LlyzMimM1W2hnmcMKwR50lrRE6iYH5w218ITtaaJyJqVx3yQkeuA5LmDJHlMcA9QG5naPKL8nBADXmmCWaTOaB+xyCEM9BrCeaXB2sb5mdYU+GhIOBDaA5TojiDwsMGhR0SyZBeN+YFHc4pMxfcCsVqcC3qOY0/D33QKqpOSDvs3HO8ZxY3cQj1v9r4WK/8j7ULxTXNUV6bnQpk7kvk863IkLn8eRb6R4niRDlYLyDS4+R5AjUbcm8YHeJnakAmwvCI7MfcIbVQn0dmka9xIzkfUbjIm4UJ3dEdzvBbkEmyCWYRla1zOVw1GbydybYc22nHhieffJImJydp165dvc8qlQpddtlldM899/S9pt1uU61WE/8MBsPpgRey5ols3RsMpxLH9cU/OTlJRETj4+Pi8/Hx8d4xjT179lClUun927Jly/FsksFgOIF4IWueyNa9wXAqccq9+m+44Qa6/vrre3/XarWlTSAlonQl7YQiE8Uc8x3z4Partd07wJFFBdBiLsjvPVu2HO6VX545xNco3fdsEbTUfdCXB/qtuUFeU5gEk0AZ9LYVNYN0TnYaNeDlQ/l1yIAHAiIzB9m7d8yV7t5oikiOkmEqAj1opNy7RfQIVoIo8BwuZKVzu/I+2G7ULk8yio6f47bHIfcXesdrr1mkX3MzqNAiJ5Gm03uf6wx4QBUKShEeHcVfiGQmP6SrM3U5fthWtH7ojI/LGbmIpPkoBg9/bdryOglRR9mQ1hgGrfvEJ3J8oiQrPd1FBk+cUqhRrkxy6CWeKMd5BM6jJMCMiDBGeUX1Q3RKOAUmgSLsB2osF7eyvSE7ze1ujci1FIIAD64R3Av1s4o5Dnvk4hbZCBS/SeG26OGvzQjuInckerPPHyzL88C02HiSj32re36vnEzLdZ6f5L3Ub2IUDElAk0bA7oP7js7YiuPq5LjuFDIEkpoXl4493St/8/BvcF2hHP8uvEeSeWarvEW2eSa+jGAQ44eZFrU5FTXEIFKlcQ7v7cGiigQoLrUvPRWZ+yYmJoiIaGpqSnw+NTXVO6YRhiGVy2Xxz2AwnB54IWueyNa9wXAqcVxf/Nu2baOJiQnau3dv77NarUb33Xcf7dy583jeymAwrAHYmjcYTj+smupfXFykxx5jL80nn3ySHnroIRoeHqatW7fSddddR5/61Kdo+/bttG3bNvrYxz5GmzZtore//e2ruk/qLv1rqdw7Xgc8IyPgbFGQQ9E3KNIz3+jv+U1ENDPLlPLTEVM2ImEIEbUWmK7K+kjZcrnyhEomAtRcfGgaDkjbJgrctEa5jmBWNgIThWDZn2dXZp2oBGm6zAJQbHVJvwV1vq8D1HqwMDgaQSRIgeQyKKJBJKnLOMvTL5yV9RF44SJ9noDpIVgYzGt1kSJXHuIBinfAc3hteZ4P9w2PMPcY1AYvm+wM1+Ed4U5BCphIJnApPwUTVrF+4Rw/I+q7++B97qqMVX47JdKiTi8CJ2vNEy1Rt05ClKrET12QRkfPckqBCo8He7CjeSdVCXPQXBMVeGyxvsyimsdt9LwHar6D5itFmcPcDw5hmIIMq0lzmDkIijhtFR2PSX/aFX7w3KQyA8G8qG+Cz1tAiyvzIyabwWNOR/5uTHL9XcqrRc4iNBVI+hsjGpDe71TEaRRPc/s2BbxZxLCH+62jiGwt8juAwPyYNKXd786fvZL/gOoy8zI0NfUgERlGlpT4cz0f3QjMA5Dcy23IukUCJdhLs1M88WvbYUEQ77k6WdkgrPrF/8ADD9Bv//Zv9/5ettNdffXV9OUvf5n+9E//lOr1Or33ve+lubk5uuKKK+g73/kOZbPZQVUaDIY1DFvzBsOZhVW/+F//+tfLOHQFx3Hok5/8JH3yk598UQ0zGAxrA7bmDYYzCybSYzAYDAbDOsIpD+cbhGVbXzg3+JxOjHZc/tyv6/P4MTFEJzsv7XZxju3eeZcNTjpMhCAcJFPnX0LBAtu4Fs6SF/lgQ3MrbJ/RIjYYzofxWxiyR0TkzHLb4wIIi4R4jbRlos0Tw5YSJcqBfZmGGN5EAzEoXErbItHGj2EsXWmyIoJwQwwvFCIoZWkvzNS4rcLeF8txRl8MFCeRKeHk2ER5zBCHwiCy2cthNUREKdhddfikA7bN1hCGk6nwQAjbQRu/A8/kKh+UxHcoOdpgrWFk6kReROQtyofy2jzWmIUvbfEAiKx7ROSO85gF8/B5XZ4X1DgUOJxme3RzlCdlZkGGhDqLbG/FDHj+DG8+2Vm5sPw6jNksh4C5Iyrsa5bt0U6T+8HbBuFuTeVDBHZi9InBuUpEFICpO7PIx1rjqISlBGAOced5TY6+0L4A3jyETQ9zfYWAn8HJS78jtIPLrIKy7vwUiBrBidgPiSf33Az4Fzk+zwVvgdsZj8hxLQ/z+NUfrfbKUVE+rBDqqnNj3Q7fU/uQYUiqsOsrNg19qTB83W104By5VzWfDwnFMOGjwX7xGwwGg8GwjmAvfoPBYDAY1hHWLNUfh0SUJVIJ6ygD2uUxUP3ITkWSOaMnZlmYJ4E8Id5+SbG0u9wdBYdpFaTziYjCCtM0TsI0IYZopb7sWkHNQXiZDiHDY5kFyNyn9dzbIMoB2vWYxS1V5oFwtr8wiK+eD8OQnG5/6mpFljQQOsG2pgXZD3G+v01A04ZoBsCxTfKod63rGOCAFsm+Q/NDkgM6UF2P4i0YXoZUnBbDESE8EJLoteRERtMNCtA0iiTPg7FACjcFM84KgSOHVoQFni5wkiWLizB5kdJqBytAChtEVJIhk0iTCgpUhXfimEUQjoXzXWeq9CEcC/s6GuUBRNMMkQyzc1JObtQclWvCq1f5PFh/wsTUlZM/zUAYIoTsCXEiIkrnuIzZ+siHENwVuvb8N4Y2e205yaIC7F1HuD3PlKt8zXPS/JFAhSjSg+GFS8e4voWEo0UaY9x37RHZ7EyD+6iQ5fti2OHERhlHfOHIwV55b31Hr5y6cixjmGpukcMdcQtRlkNhHogrfI03I3UqcE9BEyOGMmssi8ClOqPjoPOP6SyDwWAwGAxnBOzFbzAYDAbDOsKapfrJSyn10hVe/UIoBjL3IW1VUgqfG8r8Qe0pTgkVh5JOareYVtkEFJT2zuwcUvzZcn1Zbk84rygXpK6b4AWqMqw5Qou7722WjgF1hVRcuIM9cJ3/rak48D6eZ8/mOJSJVtAhHDP3hehtquhSzAqIGfk05ewtgke1EKeQ9hlnnt2Pc9M8Zqg7nmlK2hCPtcs8RsmIzAMfY0Y1eNhUiWWgKSGc5PbE29l05CkxHIwEcMAckwSyDZh5L9nGzyGy0pHUIcf2oOc3ihgRLdF+SbS2RXoGoV0h8kKizpi0eSBNKgShhjjDZnwUIRXMwoZrTAO9snXGTgEHzQBAUeP8bEjTA4pkZQ7zQDuJ8uoHgSoCCj+oQQRRTWUSBM9yzM6nM2wijewvQP9A6FKrKq8hyEyH4mNxqBXG8EZQNZhkI+VFj+YYpPcLz6osjNDUb02yeE7xWTAVlOR+EM5z+zD6IzjCC6l2UKpK7r2A15L/GI9LZmFBnDf8S9jjwIyEnvdoKiSSfY9ROWi2JSIK58AUjFlZj/C8iApVcU3n+fdi3Dk2G5/94jcYDAaDYR3BXvwGg8FgMKwjrFmq34kcciOHutrLGejOTMA0iDePFJSkO/bPMB0YbODvOpWnJOXneUyxfLdxDl+jtI9zG5lm7Rb6y4k2R+R3KqTmHPQCVUIe8TA/cADa2aRFOfAaSOCzOM1lpyvpqQDMD50RpvddTQsLqh+oNDCz5FVCmgwkBkkCPtYckfYKvwGCJEdho+ONTKejFjo+azenTBllMFngM3QlLSqo0AyPbXtILoduHhKcbARvbRjyYF6NC1CX0RibKLolWbfb5fuiKalTldXlQc8Jva2TKvejTljlN4+vSM/JhBsv/fPrkhKO8iA+hcsRRHp0dIMQlEHquSSpdex/nShrIGCcO0MgpATJeBobZF3ZWZhrEzx+qfLW9sGMh5E5mHiqRSpiZ4Jd2ruwXjJqbqA3elSERD9j3N/u4yrxF5gbcof5msVXSoo6BUo/gaRGm0bY/HigKV3vIzCPol59u6oSD4FZ4rKRp3rlH4YcHZFIy8pAU2l7K9P+2aJ8hpzPY1k/h5+nU5GVN0f5WAkiuFBkTUcdNSYwUgkOqggwjP7wOlyf2+E1gFEORMT73TFG89gvfoPBYDAY1hHsxW8wGAwGwzqCvfgNBoPBYFhHWLM2/h6U3Q5DJNoRf28Ja2zc0KF0tTrbZzIYaaayHMVQXyvla3R2ttYBDvnIgvk4KrLdBu15REReC2yRObYD6uxOIuwP76va6nTAJtcEO9kC153kVSYzaJLXgux6StjBb0AYTIHtShhS1R6RdsDmGJeLz4GNSmUBy0AYIdo2MwsyFMfpQEY28NnAcCmvK8MQXbDlC6GhjPIzgLFwF3hC5A9KZZDmaB6ugbBGyICYKpGl0jPcbvcXT3Fbyy+TbZhmpwi/BQI0kazP68C8EXMNfDmUyEccOhQfo1jHWoPbWYqOiooqmx1EgQpBKMjc56nsll3MEokRbsqmrudor2pwBXA7cj07kA0Sw8P8Oojl1OUzYBhp9iD73yy8fEich6FeKRiKRXZSFWLsNmBdubw/taviNArnQBirCnM1wL4bbONvbISQ4GeVCNHL+Jk6R7gNOZ/3Kv+wrDt3cI7vup37wVERlz5kOL208HivfHf2db1ya4MSu6nxOBe6IBTkgX9GW74Ct48d7pUfOXh2r5z6cvzR34KaPOYJ7DWdshLpQZ+TBDO5qtBMnKuwjtvgl+V19Pvl+fseLQQVYL/4DQaDwWBYR7AXv8FgMBgM6whrl+pPn/+nGEukQaolpmYXcxxupal5x8VYnsG3dIECagOfqKnwdIj5FL/FdFf2MH8+e7nM7pebhjq6IOajQj7iPISGIJOms8qVmPbpVrmcZiAT1oyM5YlDDiFCKhWFj4iI4iyIibQhxgZO0xkHMcMiHkuKcjC6Fcw4yEVtdsGwGKSyMZwv0REtEWS8gixpOpxPAG7bGpPUJdKpOAdQ5Cc7IydUuwLZG1+6lesqyoHubOSYQHyOSFovqAP9FyzA8zV4XDDUjYiW1szpyfST1yHyiMhf0NnMQGQFujKe56ycWmQJM90JrfcjKsx1sX+IqQ+WH0+JBgnhGhSrwTZoCwImEsxgGNuxhV7ifCw+p4SnQp5EKCrmK117zB4YzAAtDSI9cW7w5Aln+LzmuGx3c47nITLjzQj2Ul9eE8He1YJIv2Ce5HmwJ70qPNQrI+WdOmqvgfBj/6xNvTK+DzAknIioE4NpJc/H2hVl/oA56AxX+XMIZdbj3y3hhgf7SakgzsPsn3ifzDy3pzEmQ1KX95DkKO83hP3iNxgMBoNhHcFe/AaDwWAwrCOsWao/CVOiMBV0zRL478VFoDuy4KlZkteEOabpkoCvaQ1J+nV0aKZXno/B815RNmGe62sNAb21ielIVzGDMVJAGdRYPgonixaKQH5Hy0yDd3sANN8cUOFlSQehCA1mjtPP57XR4xTpc2iPyjjYBbYKaXtNY6LYBnrEJ4HygAXaPgFqPcm4fT9fqg8ybeEcUFra3RzUUWDK1K8rnfYu92uU47mClHJjXNadn+zPtWUWBpsbsC91pjV8RjQ3YBtSbQ6L+0SLnCZI3SV6Mwnl1tQa6S+440+wyEpbmcOQJhViRxVJrQpzClDMaC5a2CavqTT668gnORC7GZJzIwviK6jvHisP/RTWc1QFASeIaMI5TETktJGWxrrFaSLDJsKtDzY9CFEjPKT2DX8G9jUwOS6AOVSboGJYz/lJMDeoPslA9tT/evj1fF6AkUHymvJ+bncyOwdHeM4UchAuQkRPTYMAV4iCTXI+YtRBCl79DmRB1WsZRYgEZqWqXJzhECm/zm2ICtwGHcmzbH48mrAbwn7xGwwGg8GwjrCqF/+ePXvoNa95DZVKJRobG6O3v/3ttG/fPnFOq9Wi3bt308jICBWLRbrqqqtoamrquDbaYDCcPNi6NxjOLKzqxX/33XfT7t276d5776Xvfve71O126V//639N9TpzGh/60IfozjvvpDvuuIPuvvtuOnjwIL3jHe847g03GAwnB7buDYYzC6uy8X/nO98Rf3/5y1+msbExevDBB+lf/at/RfPz8/SlL32JbrvtNnrDG95ARES33nornXfeeXTvvffSa1/72mO+V+ot/fN0JiJUXQPjdAJfYbRKVzaA8DkM0VF2+JlFDsHbHLC9n5Q9pdPibitA6Ew4y4afha3S2II2mbTDD6Vt5Q6qthVA6S2vss9VwH4PVUTbWjQImPEMbWP1TTpbHISkVblPMNQMw4JWAKOblA9DnO1vhGqNKpsV2OtF6AyG/R3FPQJDmiiSBu846H8h+g8QKTsa+lscxS8D7boFsM+ijwCRVEQUGdlkFCjFYB4V4YvQD90SCSQZh5LjGM93Mte9kz6/flVoHtqmvRb0XYJ288GZ0sQcaspNxUV7LdSBKo7hjNosIHMmhsyhH40OLxTZJEF1b4U/Boytv8j3yTTYfyeoqUyCDcgY2ED/D7X+YO57ot1cXuFXBX3SHAMV1JIMhXMg82lmhssLdd444rxst1/nOlJPhtOKdkP2zlfkD/bKP1t4Za+M+xaR9N0oQD8kXf58bl76bhSK3I/tucEKpsECbggwXnPcqV5HZRaFrJzuHExoX2cWhVDRFvePD3uaVudb3hfjYwwNfVE2/vn5JU+Y4eElh4gHH3yQut0u7dq1q3fOjh07aOvWrXTPPff0raPdblOtVhP/DAbD2oWte4Ph9MYLfvEnSULXXXcdXX755XTBBRcQEdHk5CQFQUDValWcOz4+TpOTk33r2bNnD1Uqld6/LVu2vNAmGQyGEwxb9wbD6Y8XHM63e/dueuSRR+iHP/zhi2rADTfcQNdff33v71qtRlu2bOmFJCWKGXaBpYsg3ApZPh3SkAeqvy5oaHkeivTEOv3fACB9HhX4xp6MEhEhMmmD6SCdsc6fx9AQ5n0zC5JW82p8XncD01VpImwhshF4CG4bKFGjVJhNIDQPmqDD7/B5XQgHREEcIqJgDgYQQtXCI3IwvCNMhaUuPx+G+Wkq1YUQK78N4Zh1mb4sOwdiPgEI9sSKKgYKNtMAQZOI2yr6m4hCoGAzhxu9chxKPj4zD21t+HDe4H7NgHgSgSkknBWXUKYeHz1b4YvAiV73mcWUvCBdGa6EmfcGCJFgiOrSeWAOKUC/KmoV14XXBHGoCV7cmoTGjJa4j7ggFCToYNWe1MMQQq1IA2Yu3K8wrFWbq6A+7KsVmfswESeaP6Dr9H7gNHkS5qYhc99mtUcOMK1ks7D/LijhIshUioJCaOYkIspPglgNbLoY5uwpKydS5o4He/MRuH5UmnBaLWgfrG2/IddT+1wIrYO+jyHjqH4PZSBhZFpAobbB6fYSMAl5jWjwec+bHxO1Hw3CC3rxX3vttfTNb36TfvCDH9DmzZt7n09MTFCn06G5uTnx7X9qaoomJib61hWGIYXhYNuOwWBYG7B1bzCcGVgV1Z+mKV177bX0ta99jb73ve/Rtm3bxPGLL76YMpkM7d27t/fZvn37aP/+/bRz587j02KDwXBSYeveYDizsKpf/Lt376bbbruN/vEf/5FKpVLPflepVCiXy1GlUqFrrrmGrr/+ehoeHqZyuUwf+MAHaOfOnavy7CUictsOeZqqJkmr5QvM7TQ88MBUno3T8yzgg/oTms33QFni4uzTvfKXVIa4pAtZroB9Qc9Prynb0CnxNdkst7Vdlo3ojDGtnYRAs7ePQt0ixQZ0daJ0x9Fj3AdKM6ipCAS4FVLG6C3sL8r2dMpcB3rHI91GJMWBhMa5Guqkwo1Fahe9/VdEDBT4F+TiRm5PqSRd5Rsb4HlhALtFZW7A+4ItqTEBJoo52W7MmhYNycyJoq2Y4W0UPMEVm4d0YXOYr8ke5L5b4dXvO5TodH4vAidz3Xfzzkoam2RmSBEJA+YeryFtAIkP4lW4FBS1ijQ3jgvuI92SnBtBkcfWb/bv61TtG0KDHdq9MrKH15YL16BZUe9dacge/14b9g1lFsFIBZzTnTGeeM4TWv1qwPxUW1IaYOY9oNl1alAA7hUya6LK3Nfkm/3fuXN75cYGMBUMyborT0J/w7PGw/wQmyaknWy+wePaRdG2qvai53La5k5GETJhXiK5/yZ5GK85KRrlYr9i9E6Zr+nm1Tvp+ebFx7jsV/Xiv/nmm4mI6PWvf734/NZbb6X3vOc9RET0mc98hlzXpauuuora7TZdeeWV9PnPf341tzEYDGsItu4NhjMLq3rxp+ngb27LyGazdNNNN9FNN930ghtlMBjWDmzdGwxnFtasSE8apJQEqdS6Jik64QGdiWySps4u2/pUr/zgQxfw9Uo0YfPwXK/8Lx3Wb9bJG8Iie7kmPvNvqBndHtLe3lx2wKs4N6NpRxS4ASpuOBDnuS20MUBxHqj0lnRzdQc4hWrKPAAa3+lEA89DZKf5WAAe680JmcQCvWPxWSPFirugf46Uqws0qDYj4DHUYtfUbrAI9DAKh6v3G1KrSEmiF71IFKSAIkQ6CVQMIis+O/+vSD6VO8LPhP2fZIGSVnRulHMp8l9wpO4pRRIQOeFKMxVGN4ikRhD/H+c3E0JE0hxlLETEAKyrKMdrLjetwnTE3IX5tAB7gydNTLg/oFmivkVO/vyjUMcwmylxfq6w5MQ8T9A8VN8qJ1TlV3xht4xRImh6kFWnOe7wTgVEo0I1WaFRKdRXzPLztEpyHwunob9fxTfGNUEk5/4bhn/ZKz9T3w73lCbLxhiYxkBIJyhw3x+eK4pr3rr9kV75H+Yv6pV1xA/2MSZky9S4nDryWXHeunOc9TJdkMpJaKpJgv4Jz4IF2ffNUXdFu46G03N3MBgMBoPB8IJgL36DwWAwGNYR7MVvMBgMBsM6wpq18Xt1h7zIWWG/xKxSszOQ2Q6FNpSd4/89weEfRTCnFA7JeJTHn93A9W2BTG0qPHDH+KFe+WCTbUTFp9luM/1KGWOVm4YMUZBFSod8FJ8EW2LK9nG/Ltuags05gcxRSbZ/djciogxkEuuUBmeYag5D6CGI9KCNSkfoYKgfIs5oHw2wbcMzxTl5PdqwOyWwHUJoUTenQlpCyKaFWf1m5sR57coY3/cw97HXkX3sxNjHfC8My3E7g0N2UAhJCwCJdsMq1PbV1hD4g0xz+zzwgdAiL26UrvBLOV3gN1LyolTY2onk3MvO9Be80kC7sPD58ORYYHZJzAyJfida7AZt6ugrhOI7oRLSyR4BsbAMnDcnnzUp8pxM4Dxcc9oXJIGQVTxWelyFC8O25Nfh8/AovwHBR8bHda4Tg4JPUmaB6zuryHF6k/uHxTVRCbLegUk8VrmdvFb/9mFIYnhEXQOZEl1IFhU/w32Ve4nUiPj6L1j0x1nghZmpqTHyYa8JuOFR2D/Ue+kDKKM/wsYxcVp9AvyJ5mCuQ4cv2/R7bYjl/78O9ovfYDAYDIZ1BHvxGwwGg8GwjrBmqX6/SeQlK6nPLtC+DmRWQlpF00Qv3QjUfObsXrkxKjnuXIG5rxGPbQLNEXneT5/gsCFkXJobmULKHpY8GGaFC5qg2dxWYV55EAaZwYxZktsTIh9HIPbFxxRn8ppOGUPumHZc2CKfrzAFwjVAuWLoGtLYREQR0O6Y/UyL9GA4H2YjzCyq7GXwvHhfzNyHVB4RUQw0m9BmH6qI8wTtW0HKXD4TmkBQAAYpUk13Ig0dHuJx7myQIVsePLvXGbwMw3mgodP+/aBxOlP9nbJDXuhQnJV9Es5CFjUIlfXG2DzXqsjNAud7F6K2kqwMs0Ld9gDWH1LmrVFZdzjFdWDoWlTizUdn18M9AE0ZmNWTiChT47pRwAX3wliLZB3iGNNkO/P5zQk5DzDsFsOFgyNAUUvdGnJaKpRx+ZopOUadUTDdQdbRfdNMZTuRnreYJhTqVirNGPb8VGu0V/ZAEAxDZIlIhD8mEM7nQGheuyXH9awNc73ygQbfR4tGiTDelNuAJrjEl2vexW50oa1daa5CcSc0Wfo1EEtSomat6up+w9svfoPBYDAY1hHsxW8wGAwGwzrCmqX645CIwpV0GXq2jm5gPmhukmkZLW/80tJ0r/ysz1S/psuqBaZmD3RH+J46m1qJKZd2lam9TJ0b25UJoYSHcQrUmRbyEN76SO3l5VD5C/09hF3wRCXliSwoPGSaioo2fBqoqzk2ecSZMtQtLhECGfhMmK2MiMivQz+ASSBRkQVxESlT1CFHL2D9fECLo1d/RglswKF8mSnABM0kJCMSkFrHjIGRiixwwAzgYtZDNc4+UPE4x1tjklIsHuTrWiAUkpnncresIiI855h1udcasjMpeUEqTCFERBFEv4isdZiRbV5y1H4LJxWYzZQJDM1twSGe7+EYU+5BTUV81MGMMwymMZiDem1jZkmc09kZ2W6vBnMyh6JWUJcy5WB2PZz72enBkT2LW/jzBLLwrTB5gQAQZr3sjKqsox2+Ds1klRw/z6InN0ZsawYirhL1ZsIx3whqPsKLXe3TuM5TpNM389htGgFlICIazvICfq7IJsJE7SFocqYMiOdUIRKrSQIYuYRRS96iTFOIkV5ovnRgv9R7X8+scYwWPvvFbzAYDAbDOoK9+A0Gg8FgWEewF7/BYDAYDOsIa9bG362mFGdTCuZUdjawezfaYFupsLEnnJaP9Yv58V4ZVZ90trlak+0zbys81Sv/dxVyF0FIShHCt9yYy7lDgxXAkjrbkXKHpX3PARthVIBQjkUVYzMA6QjbslDhbqk+gmNsfy7tV6GCbn/7MCrFeS1tUOMi2mfdSBrv0TaJITKeSsDmT7HtLdwANsbD3HdFlQ3PB9toHPBYOvNS/cqJ2SEBbeHCL4Dk/PDrqFIIIZdz8hphc+xAqKCyvUUFDO0Cm+whpeoIWd1SB7OCqf4HdPMuxZ3T8zt9p7QUzhcVZJgV2qYxktEpsc1Y29Qxy1xrZLDPA45NVOEQLLQra18jVILLTsG4tHG85DXdImTrmwYfgYoMLwzANwfDdlGZNNNUviVN8BuCQ13ptiIyEGYh010cYIZANVmhDW0IkXSbqlPwT2jDdA0aoReCaBtUpbY7F9bVzxY5nFpkHVXNERkZwQ6fHuSOfHphg7jmYJnt+vEM+034C1LptPAsvGMwnK/VXyWRSIVnow+LKxtemIR9Fs+DrtPKpMs2/9Qy9xkMBoPBYNCwF7/BYDAYDOsIa5bqD2aXKD8d1oEhLZU802UNUJ/Q4giHF5lqSiGZEmZZIyIqZpkuG/I4C5+m+bJZCKVDxg2qi7Mqc1+BK3EL3J5OWT5gBuhKFP/QoXl4MwfCxqimuD0AZpxLgNprV1VIIYr+VLi++kagN2cU1YgiSSikk5edl38OhXmA7lbJwdI802ytCgjV5JmyQwEbIpnxTGT1U6E4OD8wc1d8FKESFNnpQFRjlJf9UH4KqD54Bp150QOTTgyZ5HQ4X+MwiIEABduBMFK3rcYiTVbStacLnKV/Tizbj1n4MEwqBXNKorJJ4jWCtg9UxjnI3JcpZ+Bzvih3WJu2sEIudsswLop2RVNEEgzeevEYMuP43CsyM8Ic1/dFYOhiexjCbgv8fNgfREROq78QEmb+IyKKSnzjYIbXZqcBmQhbOm4Xxg+sO6nqni6ss03ZuV75FxAmrUOC25ARsVjF0Dw4yZf9eMnW/b3yPXUWd9OZMhsgpDMM8eNo6tHtQeo/hXC+hGSGP9zX/CbXHdR4HAIl7NQpLPWxifQYDAaDwWBYAXvxGwwGg8GwjrBmqf5u+Xmv/nnl1Q90CXr1k4tpmmRdKfB8SClrr9sm1PdQm0/UpoMW0MM+CAIlkOlNC8ggZYNenJpWy02y92inAnSQojE98MoXWe7yzPVEVUkhoVc/ZgLUmQnRtBEXuE8wWxRmIiQi6hZBwCc72EM4KmImMr6x9j52IMsZerBixIHuY7fTPzOaTuWoTTe9eyZ64sAxeA4fHHy9prxGmDaArkZqn0gJCsGcDmaV2QXmNepxo/dwtyKXceo4K/XjTxO4naVfI45Ovwnd6nZRfQXGXD0yip0I05syuyBtjmWMNHG7epFg3ZD5DwSz2sNlGgTM4peoLGxODM+EolRA04s+IKK0wGs9yg4eezRNac/5Xl2Kok5KXLcwD1RVBTAAnTHen956wc965TsfvlBeAs+XYHCDegTce363/ONeeS9dwefkZJ8IM3EE0RYF2NBVhssfPc2ZXR0wA6S+XpfQ1Dz0zyiMgzIDRhCVgaYipy1NKTH0A3r1Y7a/5galXrdK2C9+g8FgMBjWEVb14r/55pvpwgsvpHK5TOVymXbu3Enf/va3e8dbrRbt3r2bRkZGqFgs0lVXXUVTU1PHvdEGg+Hkwda9wXBmYVVU/+bNm+nGG2+k7du3U5qm9JWvfIXe9ra30U9+8hM6//zz6UMf+hB961vfojvuuIMqlQpde+219I53vIP+7//9v6tuWOIRkbdSrAE90+dnwVsfRCY6FclVbR+e6ZUPxNVeOQ4lFZMPmXLZH4HqjKKdwpCpIhS4KULihdmXy4YXnwXxjg7fJzsr3TD9wyw8lAEPfZ1UBxNAOJi4JAIP2q5y8US2EmhtFIUgIspNw4lAaaKOfTgv6w5qTD0hPRUqU43X4L5Dj2U0FSx9AHQcJvYAE4erPL/xeT2kQn05Hxpj/H23XGJX6XBONgG9a4WYCFDIen7ifBCU7QrBJO5jH7S9m2PiNAI9EsrDuKCYjN9ADpEo04hFIqgXi5O57qMcURpKsw0RkQce7cLM5GNkyOBEVGgiTPKSJsX5H+X4REyYk6hkUWkBEkShvvsGjgZqDstrSs+AeQ7Ed4QZkKToFiYyao5wfa0htXcdANOREH2SdWdqYGasQoIbXEpqvxNmNxgHb1Guq2SEqX/3CLf7m/e+mj+PtBkCPdX50zhLA/G1GteH5r7MojI/cqAXOSGaheFzT86zjSDa88wvOfFb4kmzhkjQBus8mGMTcbCgokeqA0wwgZyPaJbqlPhYsDA4iVvnedGguHNsJr5Vvfjf+ta3ir///M//nG6++Wa69957afPmzfSlL32JbrvtNnrDG95ARES33nornXfeeXTvvffSa1/72r51ttttaoM9vVar9T3PYDCcGti6NxjOLLxgG38cx3T77bdTvV6nnTt30oMPPkjdbpd27drVO2fHjh20detWuueeewbWs2fPHqpUKr1/W7ZsGXiuwWA4tbB1bzCc/lj1i//hhx+mYrFIYRjS+973Pvra175Gr3jFK2hycpKCIKBqtSrOHx8fp8nJyYH13XDDDTQ/P9/7d+DAgVU/hMFgOLGwdW8wnDlYdTjfy1/+cnrooYdofn6e/vf//t909dVX09133/2CGxCGIYVhuOJzr+mQlzgrQ/OgxW4G7FeHQchFsYYzTba7YciINy8rnzrM2Z3unXgpn6fCxhbnOGRjBOqIIeQuOy3bgHYzx4OsVmX53SuY4DZgWI0WZXES+Bts3S7Y3by6DBPJgH8E2sc9JfggQtdmG/A5hAfqzHAodALhfDpzX1CDrH6QuQ8/J5JZ73SYVq89KswyCbk+DP/Rbc0d4b8P1dmYWCnI5SAyiYGNHsN0fBXO58LfCWQZ7FRVhsYFCMcDvwCdwTA3w+OcgC8Hismg3wvRkg08GdRpLxAna913Sykl2XSFT0sMUw+zk6UNNjrrkNcYxi9iLR9yW0oYCyr3IMQ0DgaH0+KcSlX2t2WgPZxI+oyEMzzQUV7aytGmjhkfce8LVdZRpwlrHQ4F80rICvSqOsMwVzOQCTSS1zgNjl/FzKmOytyXQla+GLL4FUZ5D6nPyRBj7NcoD5+raDUMZXx39b5e+Z9yv9n3eiL5rCn6KhU68Lm85tWj/AV0cpydBPyGfNZOBfYnCBXEsVu2u/eOoS7PIsQEd+VGhsJf6FvkQKhpsCh9E5azr6Zx/7moseoXfxAE9NKXLr0UL774YvrRj35Ef/VXf0XvfOc7qdPp0NzcnPj2PzU1RRMTE6u9jcFgWEOwdW8wnDl40XH8SZJQu92miy++mDKZDO3du7d3bN++fbR//37auXPni72NwWBYQ7B1bzCcvljVL/4bbriB3vzmN9PWrVtpYWGBbrvtNvr+979Pd911F1UqFbrmmmvo+uuvp+HhYSqXy/SBD3yAdu7cOdCz92hIsilRNiUdW4JZ5sIsUzYNyAiXKFGWcsi0CrLfWtAgzDEFeA5w9f+swv4KFaBpUqZzMZtWIiW2Bf2WxhjupsKWIBQOM1ElSiQiBho5RRpyDHS5M5JCxAyESKXqzH3yRhja1/96XYcPWeV0aCWGFB5N7xwzW2FoHmav0kIuboPHT+hVKz4P6bOoDpkElfAJ0poYXpYF+l1T/SL0sMUd7jW1qQZCI4Haa43K6vwGXIfmInwm1YQ4dCg5jpn7Tua6LzxH5AW0IhwxmEfKkz93cJ401YJ2eGwLz8D1bRWKughjMQlhkps5pNdfVOaBBq8zDOn0oQ3aRBiAeQcp4UxtcJgWmvjQJKfDQ9MscOMY7diggUCq3mvwAlyRwRIyjSLlrmeY24JFjMkVIdufW1OCWZ3+AkCYHXPpb67wb+Yu489hXRWelS0K58BMNjPXK3uPsBm3PSzn2f9Jz++VuyD6lWaUXRGzehY57Br3ZREWTURxFrIUFtnE6DZkH7Qr/BxZyOSJ2QPbZRVK+fyxJDm2db+qF/+hQ4fo3e9+Nz333HNUqVTowgsvpLvuuove9KY3ERHRZz7zGXJdl6666ipqt9t05ZVX0uc///nV3MJgMKwx2Lo3GM4srOrF/6Uvfemox7PZLN1000100003vahGGQyGtQNb9wbDmYU1K9ITzDrkhQ55ivJBSriTgI72PFMfmFmNiGjfs5yBaQjoN02zdzvcHWWXXXIdxXxlVLanZfgNoOkDScUgzeNAtrEopzKCgec2CrZoJ22kkdMYqDiPy5oORE9ZpMk11d/N9Xf9EJnQlPkDqVi8r243eimjSSD1pBuv00Fd6/70lRbOiKpMnwktbNUPnQJSkpDxsaToMzDX4LMvnsXXI01PJIVP0NSi5xBGgHSA2tNe/Rj1gaapAE0/p6ceT180xhzysg6RMm0JD+kBz9scl/Y1jIppjsKczGialMtR+Sgp4xBO/zFDPfaV5isuYua+5riMbig9ChVihk2I/sgdkXuQewQyfjZYHEh7umN2PPRUjyBzZuqpPam2AMfgGQIlilMG09Zs/1dLkpPtdoTYEX/uqcgs7MtLC4/3yt/zWaSn9hJ5TXUfX+SCiFHzLG5npipfMC+bONwr789V+UAsxwhNHsnhI1wfZAnt7pDOrREENLh1HmNnflGcFywM8x9oUhBe/XLT7mWfVOalQTCRHoPBYDAY1hHsxW8wGAwGwzqCvfgNBoPBYFhHWLM2/m6BKMn2yUoGZrwAQizqaHvKSiNgJuDzouxggyiGB5ZADixTl/aU2Q7b57KQIQ6zUGVVZq04A+FzmQEZ5ogoxnA1tO8oO3wa8gO7TTYsdxscWuJ0pcHYhT/RtqbDGjMYepb0j/XL1ORFyVaYShimp6L50IdB+CDoRIBoHwc7pwe2sXRM2t2wv5qj8J3Wld9vMTTIK6BNVtqIsV/ifP/nQ8VC3QYM2dK+Dpghjo4yztjWbg7nGvePznLmdlNKo2Oz9a01hLNEXkhEKlMlZsFDW2nSZBttOCNDrjoFHs8MmFHdurTr+i1O64e+LzocTwAz9wn1SF6L+voI/Hzyj3GDumXl34KZ+2AdZCDsEEMDNVC9UPtI+Y0BYaBQ1u12Mty+ELOdqp+NDmzC2CeVAg9e+5kiXkIuCDWJrIAqeg7DZruQwjCBfdVX6nyogJlCWLIDCoFRR74Cp2H/XJjh8oZQ9kkXfE4wE2ta5Mm5wr8Jq4BsfWlb7tOF50DlEEKZhfqoVqJ8vg16vx0E+8VvMBgMBsM6gr34DQaDwWBYR1izVD/5KaV+SlHhKNS8z9THAoSW6KxyQYCpvriIIXZERI15pmkeabJMaKcoz2vNM8VcmUcKaTB9jtRcPDffK2cakrLJTHOqrdYI38dvyOxe3gzbQLrjHL7jH4EMhnlJIcYDIpV0aIig4AOuAzPoubG8BkOivCb3d3Zm8HfLOORjKzKMYUggmgeCwVMWM/eJrFltmRkLBYqSOaCDlUkH55HX4GcK5rlPNDUfAR1P0WA6NkZTDUxPTc2KbIkY1oj0sqJFu0WH4s7pGeOXZJ5PuKdCODusXUUZji4jB0LPcD4RSdozQauQyuyJ4aIOUsIoBqTCCwnoXa8NJhjI3OaqzJL+4oD5oC0KYNrC0Doc51iLBsE6jcFipdcV1oHUs9eCMGJlviKg+qNwsBmhWwVqHUR/sj6E5qpwvvYwD0xUQDOCCtUFsa9Hmpt7ZRRi0iGzYl0k/TNganQhNLpQBQGojIyLFKGaOd5Y44JO2YoXcTEaZQGgzKIcpA5k5cOQ826Z625skHN4ea7q984g2C9+g8FgMBjWEezFbzAYDAbDOsLapfqfR6JaiPTdXI3pl0wN6Tp5TRs8N4uN/t64RETZEntX/lbhl73y1+mN4rzqBvbI7ZRYyCOoDxa+Qe9Tt8Q0T6Kyz3VH+ZlEZq2CpO1Th71jM4eY+4zAQ9hV5gFBhQEN2RyR3/9yGJEA3ssdyBwWKO16rDsCD/jmiDwPvVGFWI7KMEYDRGhi6IdImWowUqGbB/NAXto4sM/dIeYro7yMEuiCmQm96Je1r4lWRmWgiSGB+lJFXaMgkNfEOaldgeEa9PCF8YsVu+g3U3KOMYPXWoPXJvJSmbmRSFHUyNrm2DynqXWcNxmgr1Fffqm+/tR6DLS2vyjNRQ7Ss8hQ11G8R07qJANZNUNeF+ERbd+B+iAToBDIUc+KZinMHrlCgEuYjuA86N/WkJqDqDcP13crWuUMxMf289yfHGFTpNOSbufBHNPp4SyvbZ31EPcNFx4iO8P3bG4Y/DsWI6lQnIiOyMWzGPJekYLgTTCnIqQ6KpRm+XPYc4MFue9EsCf58/zcqYqcwnmH45yp8RyMirIfo9wxuvMvt3NVZxsMBoPBYDitYS9+g8FgMBjWEdYs1Z96/ZMRII2/ecNsr/zMM0z5CS12IgoLTKV1Q6DZM5LS2lBmCv+2GdYS13T8HCR2GIH2dMHzVFNVeMwJQef5KCI2SYB0rkpCswDiMpuYSguHgULKyuHtlKkvYiW4g8I1aQge7EfxGO1ylwga22/LsUDqysXENZqZBsrbb4EmOVCu4bx8PqTJvcES55SBKIYYNLf182EiE6+FXv1MY+aOSJoOnx1NDzrhBvYDJqVa4ZkM9H4C4kLOAFMI0dJ8PVZd7rWGxF/y6tdCOjg22MdphAmY5DMjzY3RQWlW0rttEELCIKIQInZQVIeIyM3xHIjzMEbg1Y/16mdIISogycl57E+DF3wJ6GIM6lBj7kAECc5bbULDREYiQRVYufS+mBa5ElznTl6aY5xZfvYOePi/YsN0r/yLtnrlwJpFs65+PmzTS8JDfB6YaXTUEia2wgQ+SYU3B6ehTJEeiIiB6aJTKYjzRKKuAvdPVOGO1P2IY5EU+Dy3I+seJErmQHKo5rBMhLTcdwPyra2A/eI3GAwGg2EdwV78BoPBYDCsI9iL32AwGAyGdYQ1a+OPigm5uYQyNWnrQzvQkTrYVooQ1jEmH+ucPGe5O9QZ7ZV1KFbosoHkd6oP9cr3OZeI84qQ0SnK9bf3CpEYkvZCzPq1uFk+X26ar4vAdhgVZH1ut3+GsEyG+yEqSaMXimBgJjLtS7HC5r/c7K42xGN7oO4BNioiom4JBDbAd6JbVOFJon2YxY8/bw3JhmefA98E7K7aojgvyo/xNSM8N9xuTpyH9Wcr3N9ok9eiT/hMuRLa8aTxrVvuLzTiqj4WAkUYztfBsEhxyVLY11HGai0jrKXkBakQuyGS4WboF5O2OMwqnJIZ0KI820E7FZgrKqNibqZ/iCmGuRafUCGAbW5fZgHGb47nU25GGth9yNKJgiv+bFOcFw+B0BaENQofAb3EdBjo8whnB4c4YiVZNsNLIR8ickBQpjDJzzCr/Ug8vq74FPf3v+Q4014wpWzqdc5i6rVhP5dmb+rmeCz+eX5Hr4zZV0v7tR8N/BHzH438aoEAADlzSURBVLnHeV02z5bzLIU+WTjI/mCbpqVaXPQq2AQaPH4Y9pmpS7+Q3BRkYWzAZqxExESG1DY6t3AxWFD7yfPvB0dGnQ6E/eI3GAwGg2EdwV78BoPBYDCsI6xZqj970CMv7BfPx8VN5Vqv/OgToOKhqM9amylvzOims189fXCkV75nbDsfUGxZYxFEeiB0Jj4qZQvVATWUnVaa3Xl+5mAeM0cpkR7MgAchgIuHmCM7a1HSk8E89wOGp3kyKZXQvnbnQDe8wH2cOyzbjaEzmEVK97FfB+oKQ6dmZNY8pPSx/xPIeKYFjpLcAPGcqoxjDGoQpgPhRXF2cDiYB9Q6mjWaG+Q1pWcg9BDovDgvQ8j8Brch9SBjmaJPRXZCEf6If4hLlkKNTk+mn1Jn6R+GOxHJsUjAROdk+bxuVV7TwfBHpH0VLY4mvwT2HDGnq9JslmnwoumCuExS5fWnw4BRfCV7AMRXRmRollcbQAOjwFhO/WZr8lrHvaG5QZkSpyC7ZQXDFYH2P0IDIUSoVBY+grmLocOZEj9PekQJJEHIscjIqISnAhDQykGsbqfE/aAz92UWwDQWAr2/SVUOwJDuZ5ogfKTCOUWoYNS/Ph3O14XMp50xnifhY4fEeSK7ImR7jMF0q8Mdl99rsd/f5KPxon7x33jjjeQ4Dl133XW9z1qtFu3evZtGRkaoWCzSVVddRVNTUy/mNgaDYY3A1rzBcPrjBb/4f/SjH9F//+//nS688ELx+Yc+9CG688476Y477qC7776bDh48SO94xztedEMNBsOpha15g+HMwAui+hcXF+ld73oXffGLX6RPfepTvc/n5+fpS1/6Et122230hje8gYiIbr31VjrvvPPo3nvvpde+9rWDqlyBqJRSkk0pnBlMvz4zV+2VkwJTZ5kD8vvMWJ5FbOrRRK/squxupQpT8K/M7++Vv6Po6qTTXxABvePDeXkN0m9Jnb2Ps7PKw/gAt9W9aLjvfYgkVY/pmnIjSCHLfsDMUXEWvcpl3UhRxhuY3sf+QsGRpXtB24AWb1flFMsdgn6AOtpDqr9m2YvW2cje9pi5z1f662jy8JtgClGiLF2ggF1IH+gr00GwAOehVz40VXtNC1OSyMKo+wuFfuB6yfpS7jBGNMD1kKVuhYf3CaD6T8aaJyLqlBzy+kSV4LNnj/R/uCirIoAyA8oFJZ6SxTHjunENe3Xl/Y0CTDDFxbo6SnRLNKRVqRjeLO8BCWSME+YK3QU+3xe16zU6FXhWEYkDEUTajADmhvmXgPkkko1IC7wnRSBis21spld+rKV15EEEDKy1rvJOR7PNW6s/6ZUfmrmoV17crKl+aBvQ8RgplmTU+k1gzUOfeG21Tx8CM0KW51MCGSe7ucHjnznC74C0JOcCmp6E6beFYR2yvmBx6YP4GMW5XtAv/t27d9Nb3vIW2rVrl/j8wQcfpG63Kz7fsWMHbd26le65556+dbXbbarVauKfwWBYWziea57I1r3BcCqx6l/8t99+O/34xz+mH/3oRyuOTU5OUhAEVK1Wxefj4+M0OTnZt749e/bQJz7xidU2w2AwnCQc7zVPZOveYDiVWNUv/gMHDtAHP/hB+upXv0rZbPbXX3AMuOGGG2h+fr7378CBA8elXoPB8OJxItY8ka17g+FUYlW/+B988EE6dOgQvfrVr+59Fscx/eAHP6D/9t/+G911113U6XRobm5O/AKYmpqiiYmJPjUShWFIYRiuPPC8nTKSydRE6FmzDspq7cHKeDMttpP1U/xbRi5go1fJZXu/Ds3LFNgA5YDxEM/rFgbbdF0IQcJMYURSlQxt6jo7G2YOQxtcFEEoXVfapUQdYH7S9jQfleTAf0C3AYGqX14DQgU7coq50KYUfAn8ho5JgxArDJPE5yaVmRCyvQ3KPkgklRIjsDliiB0RUaYB9j5oN2buCxZIAO31OJbaJ0Jk5IP+1wqBQu0PHt2r8UJIPZXmzKEVIX4vFCdizRMNXvfdElESEiX+YB8SVIhLO9x5wZycyN0SX4Tqc+6CzPDnRpyhzW1ARr4GKO2Nyo0od4BNExh6huN6tL0mCcAWXJRrJBPiJELbe3/FyqUbgzJle7CdVyhBonrhED+EVr2kNihizvHHzbO0RCAXY/C58hz0t1E2/i5kvYNhSVQTUOXTo/7+CLGMmBXvCszwGJ/FPj9pLOdZMeBnBTP+imykWHf07EH+fJQdFfR+meAcxFDrhszc6Ebs2+WBr1JnGLKHqrC97vNuAvExvtFX9eJ/4xvfSA8//LD47I/+6I9ox44d9JGPfIS2bNlCmUyG9u7dS1dddRUREe3bt4/2799PO3fuXM2tDAbDGoCteYPhzMOqXvylUokuuOAC8VmhUKCRkZHe59dccw1df/31NDw8TOVymT7wgQ/Qzp07V+3dazAYTj1szRsMZx6Oe+a+z3zmM+S6Ll111VXUbrfpyiuvpM9//vOrrsdrOuQlzgrqE2n8YonpksUjzKNomqgdA/0N9EusmMbZI0z51YGXaQ2rsDjI9obtwzCMFe0G2i9pArWjNTQgBA/b6saSN4oLKADDNF3cxax5MqMU0lODxB+IpCAGUojtKpczdUk1tYcxIxibPxJFd8owNKDwByfTEoiqTLmmig5GswTS50lFxshhCKUbcBlDsYhkP2C70QSDfbJUN2T4g8x9TiQnm9cF+hOHQs0HpPQwJC0uoblIXeMRpWr+nUgcrzVPROQ3ibxkJbWKfe4DS+oWmfdvVSXX26r2H7OVoihcTiFzXwwhmF5TTtDU6T8umPnPU6FVOM4YjuurUEHywUSUVbG2y21ToawE8xP3FDTBEcm9MQlA3OsQpqJT5scih5th+J0TKZGzUd5gWod5nR6sQRo/dU17jOvuVIHOb6nzyvz3Py2c3yujSc+XFhwxLk4AexKYG7xFuUE1Jvg8HwTPUpXtsT0E9z17S6/chQx/rgoDd9t8TfMlnCU29/i0OK9bAPEyyPYYzPGcaVdku3v73TGK9LzoF//3v/998Xc2m6WbbrqJbrrpphdbtcFgWIOwNW8wnN4wkR6DwWAwGNYR1qxIT3skITeXUG5SfjdB+mZxEby6kRZXtPFIjjmgA3B9Rnlkn7OR1SnOyXC2qUxdUjZFyPCXBEy5ejXIQlWS1JCgihyg8xW163bAexxpOUVrZ1DIA2hRH6hr9ConksIZeB8dBRGCiA1GBmDGNC2QE8wDxYkex+XBrs3oAa3NLkg3CrGcRaYTo6ISzgAhHDQxuAtSSzsJ2PPW80EXfVEORpBHr/7+mu16rqHAEXXAQ3xR0bkAjAaJjpLtCzW4/Xmeg04svfozjZSc7mDP7rWMJEPkZEjMaSJpKusA7YsZ6/RaCmA80VyXlGQ0SIwsN1L4cM/MgsrcB5S+D7S0vwDrcljOz04RtONh/NqbpIhUCCYiNGdhhkY3Gjy+zRGMGJDHBnn1+2C6a42obKldnuRogvHrcuNoN0AkCyj9xRrT/jpbavYZjo4oPMP0t243jmUFOH3ch5xEtsdvwR7igUDSsyi+I/vx4DOQLTXGDpcNz0B70jquRd5bYiXSE8O0Cw/JPQmBJlmdTbR3f/V593lhtBVZPAfAfvEbDAaDwbCOYC9+g8FgMBjWEdYs1e+3HHLJWZGIBGmQl5/F0p+/qJ/VK8dKEGW6AVQoaj4rFuXwIp/3/xrn9sro0U1ElPGAjsdjQBNGeXmNN9mfsmmMySHw68x5R0Wgk1Q/xAWmq7olKKM4TUe6uaI3dAe0y4OapLswKYa7wDRW6lb5nKxKsAJ/6uQSCPR6bg9xu7X3cYwa5dC87jB7AbeGZN9ln4XoBoiwSPOS2kUKN4aER81x7RXODxXMgxkBEwopSh0TN6GQS5xTOuQo0gOeyTphFc5/9OQNxrl/tMhSlHUoPopAzFpG7nBKXpCuFMVxQagJKNx0HhPpqDUG9gFMauMowZVgsb8JhQgGQ3l1Y7KoBMRc0DSmk+xok1qvro4Oy+gvCIWmjBWUrtc/cslXjLIP5rqZMT4RTQo6AVCa43mcO8zXzO9QUQvQDxgx8LKzWG/+sSObxTWtTRxJ1dzQX5CKiCiCNZKFjEktWBMrxMZgDaRgOuyW+RniipwLXp7rPtoKqm+CCK4Cz5OoxPuEfgYcv7gIUVk1uU8nwvSE1+MEkHWv1qvffvEbDAaDwbCOYC9+g8FgMBjWEezFbzAYDAbDOsKatfG7TYe81FkhIINfVbKQcgtFelyVtWx2ge3COWHjl4YSD4wwVY/tLtqOG0AIGJjNKbPAn3e3ye9U3SIbfHI5tjk7KrtTnAM77izYkbQmB7Q9nAXBlibXnaoMZcIuBL4JXe2P0O5v3RKZ45QIjrav8efqvAG2Z23bFoI7AY+fByIqbixvGhfYvibCnboq6xq0IVnA9F6yDZlGf5saisQE88r/o9E/ZZ624yYFXnr47DokDW18+LhRnueJtiX67ZToNA3n6+YdSkJHiNgQkeh/kSETBK90tr9usX82O41Boja4jySBXksQigohYVGV15/2NRL1ZdEWrMLnmrye3SLXh5kA9TUUc+UY6tUYl/0YJyh4xe1rj/L1hYPqWSEstQUZ69JAzvW0Dffy+vdpUpDXDApL1JlPUaRng89+HR7Mc73XiP29C/tJEfyg1DNcevbTvfJ9T5zTK3fKWmyMy2mT3wIoUOYo1SC8xm3DnhRr3xR4Bng/xCD0pftt+e80PrZ1b7/4DQaDwWBYR7AXv8FgMBgM6whrlupPMyklmZQcFbeCmuSbckz5PARfYaI8Cbx8gsNJDqQcBqVFNGK4F9JJrSGlpd6Fbhug2azbgI+RtpnKE3QyyQxxKFoR59R3NAepYqbYhjfP9cpJTvLnmB1PiG2k/WkjDcxSF85KiswH7XqsG7NnLd2Ly5jhz1F8dQJiF8EihE8CBbyC7kItdcygp0wewQK2HTKRNSXl1gE6PgMZ2XLTTOGhKA8RUbvE98oXV+rN90NmcBIvQdv6YILJTnHYWWZBmjxS59gzeK01OLQ0R1CsikhSv50h+Hyo2ivrNYJiJ5h9E7PuEUmzVRrwmGPmTC3SIq5H6hhMOnh/IqLSAV73uObQfEVElEJ4mNg3oOwp01Ea8pzEDIGp2uFTvBXUFx6BPtEiPXncOLhY3CAn7uIMiPmAGcHBRe+rujEcGobPkxL14tl/0drUK2dneGI0R1V2U9hbU02nL3+eyDH6l8MTvXIBRODIkakEhTm5ypkXk2z/sEoiouYG3M9BzKcsM2/imHXBxJB7jtszu0W+YJbbkxzjwrdf/AaDwWAwrCPYi99gMBgMhnWENUv1u60lr35H67TDV5VfzI/3yigKEczJS3753FivXAD6BSkxIpmR76nOBv5c0fHDRU4zdzhiUYcIRF1yU+ISytRRaQay11UUDQ1e4plFKNcGe6Zn6nxs5jDTTuPzUoUoUwdKCU0UKjNhB+jqeIivQe15N9J0KZeR4mqMyefLHoE/kMZUX0FRlKg5ApT7PHgYV2Ub8kDhikiFRHJu3QKflxni6I36uKTc0Cu8Ncre1bVtfE5zVLYhNw1extAeNMcQyT7HKAEdWdDYAN77ggplOjhWERFR1qHYPT25fq+Vkpek5DVkOE/q8UMO/RIyzh2Z7ZUzC6PimiJ4p+M81Bn+UNseKX3MFuepbH9Oi9vnz6EiFJhjlDmsNQImIjCHeU2lXANe9E62vwiRFoDBOZ4/hKI6cotH8bDwCKxzyLSXqLeCe4j7OKixCM3sIblenJDb4IHpb74NUUwL2juezR+YZbBbUtlEs9zusQybYdtqD0Cg9dAtcltxvJKzZN+fOzzdKz82w/Mpe0jaHlIXBIka4NXfZAo+8aWpr3AQzIpzmEZVtgEzn2YP90/Flz8s51anvNTf5tVvMBgMBoNhBezFbzAYDAbDOsKapfrjYkppdqVXP1LU9Q4kbAFNbJ1MxhPiEfy5K5lw6sRMAcXwnShRbNJhoMw7FfQKh2aqnkXxlkIevF+V52cENLTwKj6K8A1qgDsLKG6jIwb6Rwloag+1r1F0BMVRtK52p4wiGEClK437BJJQ4Fjq+rwW05XC295BE4fOagTCN0CFpllJuWE/oLO2jvKIov59np0GClgxccWD3O7METYjOEOSj0dTht/gAdDJeLJz/IwtMAthchKd5MqNiJL+eYTWPOLAIQqcFV79aOZY2Mp/lCFCZkVSG4ADESBOU3aY0DzHcYGkMZ2q3FT8aaB64fIYzDutocHmncIBHqD2qJwbWTBFpBnYkyBxEZaJaEXkyjKQIieSc7zB+jgUQ2Yz/zHV9xBlgNc7HXmeX+Z+jQPun3MrbN873N0grnFhf8H9wK+raK60P4WN/dAtqmOwByQNXotRme951oZ5cc1zdTaVdiF6q1sZnIwH99kOnKcTydXBEhXBfhA8NifOCxYwKRGYocAk1VVm6uVxPlYTn/3iNxgMBoNhHcFe/AaDwWAwrCPYi99gMBgMhnWENWvj9+oOeZGzIjQPw53GChxWd8Rl21F+StqDOgEbZCLUZBlgNyIiqnocW6Kzzx2aZWPSUA1srWBH9FXmKSEYkbI9DTNPERHlJtkWlb6a7U2uCifyamzb7I5CFqchsF+qbGNo78OseUFN2eHRNAn2S8zcp+3hmLkPbaZ+VoXzTXK/doc4zCd7RE5Fb5JDiNKXcn/785CxrqjTkoFw0QJmBdTZA8EfYRrsq2o64HnBPPdrBFm3sjOD/SjceYhPUjb+zDyPHzkgNKMeCUN70Pcihn712uIS8joppaepSI+TpEv/VPN9iH5CUR3H5w7DsDwi6bsifDH8wSFg7iLfKPF43gWzypECRHH8Bq4rXiTZI7I9nQpknZzlvSuTkec5dW5DmmebcQBz2m9J/xZngedaVGBjsvYZwRR4GD7n1yHjnN4Xj8z1iu0yZ7YLZuVp7QzP47DG93n40MZeOS6psMgB4Wc68ynuNxdnn+qV/7YGwmiz8mFxP3YghNoF34SDj0mfg7GXsD9C1OVrvJZsN4Yv41zIPcOOYwubhwnhN6BcgwkNom1ERBH86ULGwRj8OPT4d/OD53Q/rOoX/3/6T/+JHMcR/3bs2NE73mq1aPfu3TQyMkLFYpGuuuoqmpqaOkqNBoNhrcPWvcFwZmHVVP/5559Pzz33XO/fD3/4w96xD33oQ3TnnXfSHXfcQXfffTcdPHiQ3vGOdxzXBhsMhpMPW/cGw5mDVVP9vu/TxMTEis/n5+fpS1/6Et122230hje8gYiIbr31VjrvvPPo3nvvpde+9rWruk+cXwrn02ojGDrTjpDmY1qnNaKy4XlwEYbcKSq8C9noftViekpnU0NEuf6a31pPWmTHC5knWs64xNcxJYzPmio6MIWMXj7QxmkHBD78wd/rkLoSz0AyhMyFrFQIHU6E1Cw+qw4VjCpAazsYiiPrS4Y51gjD9tKQafYkVAODcwOyD+pxxrFxq0zhZhoyZAfFW1AcCEV1dLtbQ/zAWRRbUWE2URHuhWGNilHGZ8dMk+EMm6+Sl8tOdtL0qGasF4KTte4T3yEn45DTUZkqofs6Zf7DCbgf/YZcdG1YWyLEV2dKg7mcFHh+IlWM4llLN4ZwUex+zK6XHbz+kiE2I+i60wJzvZjVr1Nk/jtWc78AVDaaQnRGTA/Eq7oQgusvwnrRemAQDovmpm5JnhcOw+Y6xQc7EBbn1hUlDfMUw9+QFieSVP9Dra29chxwY6O87BOx5nC88OOqXHCjeV7cjQ4I6bR0hlX4A8xNCYg86VBfNK0IM2ws521Yw/FD0wyvCRTvWbrX89d0jm3dr/oX/6OPPkqbNm2il7zkJfSud72L9u/fT0REDz74IHW7Xdq1a1fv3B07dtDWrVvpnnvuGVhfu92mWq0m/hkMhrUFW/cGw5mDVb34L7vsMvryl79M3/nOd+jmm2+mJ598kn7zN3+TFhYWaHJykoIgoGq1Kq4ZHx+nycnJgXXu2bOHKpVK79+WLVte0IMYDIYTA1v3BsOZhVVR/W9+85t75QsvvJAuu+wyOvvss+nv/u7vKKe0348VN9xwA11//fW9v2u1Gm3ZsoWceCVdTiTpk+kGuH4C16wz983V+LwA6Kn8lKLYgAMqeiC80Jb0Sa6AbtRMy+UPc31anEZ4YQLViHrrRERZ8Op3IhbEIOX9ihn1UEM8c5gfvrVRpbKCJnXKGN4gT0MaMRot9T0vNy3p0rmXMh2IlJQzfGzepp2K/BszlqEYj19nalcLOGGUAJoe3EXJG0a5aq8c17gfMnU54VBPHb3rfaBLdeZFpEmdJuivd6WbMkZ5eCJDnDJ5wH2FpztSpIpS7OZdirvHL1L3ZK77YDFdoi2VqQKFiFCrPW3yH64S3wkW+e92BV38VUY91HPK9BdF6palGSich0xwOZgP4IWt9eHLT4P4Dqzf1oQUu8kdYPYjqoCLN1LU2pQImu7oyY9zVSOcBpq8MNg8kJZ5H2kNwZooygXYnoFsdC7XN1JijvtgWWbRFIJJ0FQdqYIRSTMRRPnA/umqTIJoHkjqdTiP6xqq1sU1+w6y8FvU5DkTleXcEllVWxBhBXuQr94bc5vgehRfysjXMEZViQydEDGCpkx57UnI3FetVullL3sZPfbYYzQxMUGdTofm5ubEOVNTU31tg8sIw5DK5bL4ZzAY1i5s3RsMpzde1It/cXGRHn/8cdq4cSNdfPHFlMlkaO/evb3j+/bto/3799POnTtfdEMNBsPagK17g+H0xqqo/g9/+MP01re+lc4++2w6ePAg/dmf/Rl5nkd/8Ad/QJVKha655hq6/vrraXh4mMrlMn3gAx+gnTt3rtqz12AwrB3YujcYziys6sX/zDPP0B/8wR/QkSNHaMOGDXTFFVfQvffeSxs2LGU/+sxnPkOu69JVV11F7XabrrzySvr85z//ghqWmXfIazkyQxJJ+9WrNhzslX/4s5Fe2VMRaOUS2+PqYJPUal7dDnfHYsy2mkQp47VbbO/L1TGGDxsq2yBC8yJQcFtQxjoIwUMlQa+u5OvQBgpltHtnD0r5QWc7Z5IKj3AnBRsGTwN/aq5XdjtsW9MZtwK4VQIhNtoWic+RlvgBs4dVBrxFzGwHRbB56fAtPIb2XW0vzh1io7gLY6kl7VDdK7MIfhl19gvQYY3ZGRgACNmJCrKPw2m2TWPIzsowUC5jeFmcx1BWeU0cOBQfLQZ1lTiZ675ddcgLHWn7JRkKhf2Aa6mr+hjXtwi5c4/iRwE2erFmtUhoFkK9YGl2hngwtA9KYwzsuhHvQytCBSOeBG4HMtOBz0luWs19CLuVYWyyahEOPQL28W7/sGQNtL07bRVi7KFjDRdrTfC9aSj/ClQfhHmsMw6ib9cFuQO98rcgFFn7JnQxvM8F1dMNXFk7kjd6+SZOPHWkyX45blv6S0l/G+5Hr8mDnvjS2cwDBVl3ERRVE9nhOlNs7zYQ0u2q/bc3B48xY+eqXvy33377UY9ns1m66aab6KabblpNtQaDYQ3D1r3BcGbBRHoMBoPBYFhHWLMiPd1SSkk2Jb8uuSqkgx6Y5NjfOIvhfCqbWpMvyh0CWk9RLGGW6ZedhUd75W+6rxfnZQKmcwZm7lOMiwjzCpgCqp0tw4Sqj0KWOqDOWptkOBjS3D4I9kTbmPKLi5IDRtMBZqKLZRMonIc2QFtR+EZnokOgoJAbq1CVkP/ugshOfbOsIykxPYghj0iLNjbK58tNcj80h7nDK678fls7h69LstzWRGdHBBawU+FOiiDcUdNy7Spk7ivzmHkdSefGIL6C4+Iqi47f5OuiPIh0QMhkkpED6HZPX5GeOCCigCiqyDBBMRaw5tw897Gek5jpUJtDBESYHPQ33keH07bRpMNFHJfUk1RvAcL5gkNsu1jcLmNZQxARQpOHeAa9/Nq8d2FWx5mXSwEYNMnhJpUBUZ0kUHMHMsth1kR/RNpUu01+3hRCJuMYsgJmlbgQhDWimSsjo+wE9rU5qyruDVFB0vbRIuz1EDLnZ7l/GnUlkAPZHicnq71yOZIxs8E8vG/muVNdeB4dWhdDv6KJgxJl6gE4YNby6twGvyXfB8tZKuPOsZn47Be/wWAwGAzrCPbiNxgMBoNhHWHNUv1O7JATO9QtKu/xeaYy2p3+zdde/W3I9uaDc2Z6WJ7nQraprxy6nOtTwgdbh1mI+lDCme2EcIZib1CnHYVFwpo+kYteE7zK5yUHjLrfeC//Kaau3KbMf45exugVrunlCMRF4iHOCLY4wfRUSZlJImRmwZu9rUSIQtDMxsgArz1YYAPryD9HA4GmGzT3oLmCSI4FUqYrRFXQigPtiYEC7pZlu4vPQEZFoPOinBYngfsChatNRG3QcEdaE80S2SPyotR1jmqKWcvwOkSeQys972EI0bM8xYgWRccLMRZcZpHK0AjjGYN4Eq6L5gZpTskchr0H6sZ1pdEtw1gu8ANpkSyk7dMcrz/hUa+ftcr7UHsYhaxk1bGog+/b3MR9kp3R6wDMAFBfNC1pcvK5TVGJO+XCcU7d/OD0S8QlKFaF2RljVXUGaPv/OPx4r3yn/8ZeuTUq99LS01x2S7zxbxrmfbHWkh300tJ0rzwzxnR6a2xInIfzEU23UY7LXZmQkbpViBjBSCM1H1tVEOOaA5PJOD9DNyfHaDmzYXyM695+8RsMBoPBsI5gL36DwWAwGNYR1i7Vny7Rcxnl1Y9fVc4eYcr98QOKVwGUhjiBTwq8qtaKL4RMsY2H7Kn5y7z8fvTos2O98ggk8EHPfe3tHYeQmGeeqSadHMgBr3VMOqKpPUxKExcgoQxEN6z4Wgd/Z+aY83NVAh8UFPLmOdFMdo772FUCFJ0q0Hx5pqdclcQEhVRQSzuzKM9DyjuDfTyIvlUIFoAC7spGiLGBG+m2IoWLXvl+A541q8YPrnFrICAzpDhXMEsIWlMPM1gIFjfyH0OLTA82N8g2ZOrpSq/v0wRuZ2maum05GMKLGi03XRC+UeYnBI6Lng9ZiGLBiBTsw6CmEuaA0JYLlDl6Ya+I0IAxc5p8MD+lTgQP9BioY4z+aA1J01Hhl6qO3j3l32gGFd77CYpQyWtQbCp3iK9Z3NFHRe15uFnu40oGbhrKa1JIWKYTHiEw0ujJLj8UCng5KlFXt4i2Oj7WTfieniuvKcPD5zPcp/pdgWsOzQjdMp+Yqr0d+zgugNhYVyYHEuM8AiJii5jMSVa9/K5IBw+JgP3iNxgMBoNhHcFe/AaDwWAwrCPYi99gMBgMhnWENWvjjwpLmfsyC9JOgiEtTx5m0Rm0r3dkIixKuvyYASRg0mE0KWSYaoNRR4s/bByb65UbOdYcd8G+0inKusNaf/EWfR7aePG+WuTFzYIdvQXZonJgC83KMDYMG+uW2ZC0uFk+IGY3DMcgTAhC15xEiW1g9jJoT6xC6VBIBf0W6puUTwSERaEdHTMO6lDBwgH0H4C2efK8TgnsipDFK3VVH8OQYTgezhs911yYQ4UhyCqnhjkqoQMHXC8ThFGnhCIk/bNORnkVWpklik/Tr/ReNyXPSYmUTwuKIXngX+JUyr2yq7IVZpr8dwfs8GlOZbQEW2x7mI9hfZ2ynO8YverCnhRDmJW2C2MIZlzm+3Qq8sTgMN8Lw3aDuaP5o6CiEN5TnoZ+Q8WnYL8bhn1DRZ6iz0G7ArbtOSWKlIFxmeYb/7jAaTmdulpjXe48Eaap2oDhuX8zdyk3DcY4/6yc9KUD4PcA62XqEfbRypwtUwT+8+T2XnkBxIUm5pRfyBFuYArhl7ln2P8gDsvimk6Jr/Hn2JfAqck2uBG/1wrPolMGP0OuIJ91OZQ87hzF8QnvcUxnGQwGg8FgOCNgL36DwWAwGNYR1izV79cd8iJnZVYyoMn9HFMs9SbHNxQPyGsOncOUTQFCsfJK0/oQiPk8trChV/YUhTizwPcqAO2IgirtquSqkKp0QMBCh6R5DRBiaED4XKRozDmmgFrjTCk7oKut6WUR0oThcgvyvNws94sHutE+UF/4rEREmUXUh4cMZXXZ7nCa290e4fryk4PDGvHZBfW5KKcv3rd4EKi5moxpcrtM9bnTbBPw2nI+YDZCr4XHwHQ0Jy4RVL2DYkVqDoWzEA7W5nYHeixw/oPut7/I1xcOyjRnmXpCrso2ebqgXXXICx2hPU4kM8aJMKkmz6ckkNegiUiEPynRJgz1CiBDJq7h3CFpg8FwPkR2iincxmhJHPObIKw1yyHG3pDMCug0+V4xiFWhqa70rI6TRRUwLur5hOa1TqX/XtEaVSZQoPpx30gqsg1OE+jvAeGkTlXZsqDv0QyB5hMiuQdsCw/1yhhmiSHTRJJaR0mbuALXLEizz4LHa6zbhax5WbWfYxgviBihCFlzSLanW8G1DM+tTU9wKxQyC2a5U1IlALRsMjaRHoPBYDAYDCtgL36DwWAwGNYR1izVnzrP/9OiA/DneIl5rMd9FlHQ3rROAOImqOtdlN97PKB5/mTL3l75o9l/L86Luv2peswIt4JqRf2eBtN82VmlT91h+iwCfspfkNwXUj1eG56vGMHnSowE6E4XvD+1KaMFwjBFaI+PZo26rDsBOhA9cDPKJOAt8nNkAvRelu7HmWdnuDy+ic+r8fXKmV1oYXeAPnWykkpD0SWROVFl/kJN8BD006NXcX1+gwRwDnizPD/jsqRz3Sb0awuElbqDKfouePJidEMkpbnJmT5tE/dRlCVKs0qvnFZSvz2A2Ux7giM97LeOrUdEdAp4jGtqNQ3leC4jzvI6QI9zIrWXxbwugjlpNnBAtMXtoLmIz4mUSAuB6SGG9ac99BOISEGPfx8EwcI5NQcH6MUjtU9ElBZ4TqddEKuJ+bykITfnJESqnj/X0S0Y4fTLJu8HuH61WcOHfTFtQ/QARCOI7IVEVIfFhO8NvZcG8/AcITccTZR6/P0FjOrgNe+pvd0HR34t2tVrt3rHLUd5pEfJfijOP7bTDAaDwWAwnAmwF7/BYDAYDOsI9uI3GAwGg2EdYc3a+JMwJcqmUmGJiBwwtbQiUKXLsW2lo0LpgjwbjFxI6eYpOxKGb/y0ubVXzs5JG1cbwgjjLMd8iexiqm7MpuUE3AYdFpcU2faDYXZOJM+LymgXhmNg1HMXW3gJ5Q7L8KJBQHua0+A6REiiDkNEMxU8a1OpiOWF2thg++zAtoHt14llI9CGin4GFCn73AJf5wxjRkW5HNAXIC6AARKqzjSkEQ77JamCapfKvOhBNsEOJPhyu3K+Y3hZF/oOQxdXqBSmtELl73RBkl1a93pshR8KPhv04wp/IDjPQ3urmg9dUN/EcDf0EVgRngb+IFERlRZ5XLoqM6gH2f+CPM+nFe0G9UBxX1iX2ueAIr4G7b9dteTRbyWCNRcVICvcYXkNqvP5qJTpq0kGYakJ7MchqNzVPXlNAs/RqcK6jORvUuyHf1v5Sa98b+s1fH1ZrrEu+EE4YIdHu762oftlTB8IanoqA6kIjfZxTwKfDGVvF8qdoM6ofYvwHYf7GGbr9JQ6qlYJ/XVY9S/+Z599lv7wD/+QRkZGKJfL0W/8xm/QAw880Duepil9/OMfp40bN1Iul6Ndu3bRo48+utrbGAyGNQRb9wbDmYNVvfhnZ2fp8ssvp0wmQ9/+9rfp5z//Of3lX/4lDQ2xR/1f/MVf0Oc+9zm65ZZb6L777qNCoUBXXnkltVqto9RsMBjWKmzdGwxnFlZF9f/n//yfacuWLXTrrbf2Ptu2bVuvnKYpffazn6WPfvSj9La3vY2IiP7mb/6GxsfH6etf/zr9/u///jHfy4n5HwJDUIazzFsdXIRQJ0Ubd9vwmJh5Sj19EPDNprtM02J4DBHR4hzT+8OojeH2LxNJSsnJMU1fO1s2YrgJ4kDwHGkgz8PwPhHaN89ti6oyzgtpTLxGP18ANFKaRbUbLraHZXuQKsQMWr6ipLpVfnYULdFjEW9g9RuksTAsrjEmQ6oKB7lPRKhmRrUVhVRa0N+acc0DDQnhgYJKzcuLRDgeUHiZmgzZwsx0MQorKQq3Bf3cLaBYkQufy3YnvkNJcvwC+k7mus/MO+S1HEmFEgmTX2YB5trMXK+MYa1ERE4B+xjjaeV52TkMmYMsbJn+9yQiMbYehMKheUCbYPwGbhZ8XlPNY68JoclgDkPBpuJBZeYa5vUSzgPNPqNClsF8hZn7xD6r2p1WeC/En4pOVm7OaRcOgskqm4EQ47kBcWhEFMzCvtGUp+FY/NPi+b1yc4xfCHrPXRauISJKm1xhsJHfG+WC/GLqwkY9NcV9qucWhtCmdY7pjc5l0bZ2Sa7BbgnmEOzFTkvahd0Y9w2wF+D+rcxDJzSc7xvf+AZdcskl9Hu/93s0NjZGF110EX3xi1/sHX/yySdpcnKSdu3a1fusUqnQZZddRvfcc0/fOtvtNtVqNfHPYDCsHdi6NxjOLKzqxf/EE0/QzTffTNu3b6e77rqL3v/+99Of/Mmf0Fe+8hUiIpqcnCQiovHxcXHd+Ph475jGnj17qFKp9P5t2bLlhTyHwWA4QbB1bzCcWVgV1Z8kCV1yySX06U9/moiILrroInrkkUfolltuoauvvvoFNeCGG26g66+/vvd3rVajLVu2UFRMyc2mlH9OZcyCFh9qsMsq0m2+MisKOhfo86Cm6BvwJEXKx1XmhnyZb+BglADQvOgBT0SUWeRK0hZT0sVnZeUoiuMkTNs7XXke/t0dYq43LmDWL8n7YDYszHiF2eaIpNe7vCnUpUSDkB70ICtgW3naomdqZh6fVXrNIsUpRHoa4PHsyox8IssZemRnBrfBAfMOuYNDC/wG0sHcVlfpxuNcceowEUuyrSg2Fc6AmExRnEbJLFCmEF2CfRfOyb4LazFFar68GJzMdZ9ZXIqI0Zn7UKwL5ysKXmn6E6n2YB5VqXRqTyhCv6GJKdBe9OiJjVYEWFd6bsRgnsG5kZuWIkveDGi6b6pys+uDqV6MVMAMj5GaTxHYs6Is9COeU1B1Q4SFiPhR8y7N41ri4niew5OeCzYQAjOIZkBLS9P26MW+LeSwA9zDXRUJIOZJgbn57jO8Xx6uyHUZlgakiFRdIqKYxkf5NAweUWZAcjHMBMx2eTn+7TLsDbB/onlPm0aXt89EbcuDsKpf/Bs3bqRXvOIV4rPzzjuP9u/fT0REExNL9o2pqSlxztTUVO+YRhiGVC6XxT+DwbB2YOveYDizsKoX/+WXX0779u0Tn/3qV7+is88+m4iWHH4mJiZo717Oc1+r1ei+++6jnTt3HofmGgyGkw1b9wbDmYVVUf0f+tCH6HWvex19+tOfpn/37/4d3X///fSFL3yBvvCFLxARkeM4dN1119GnPvUp2r59O23bto0+9rGP0aZNm+jtb3/7qhqWmXPJC12KJQsiKKC5BlPhmEwCaVQiIieDIj1AdeXl955Cjvmbi/JP98r3dC4V52GinyLQ5+jRrXXoURs6BO3sWCVe6Fb4gZGq1NRnnAe6GegyB+g2tyWpfqTwvO4AOp+kmSIN+T6CIlfeox7oQHuto9QNiWdwLDS157bBExgFjyCR0SABCyJJ02rtdKRC0w63p6USP6FZKMrDeaPgha8iIrBfuhPsFRznNTeHnvz8saeYRuGFDeOHCW5ipRfjJH2S+rwInMx173VT8pxU0N1ERHGGzXo6McoyViS5AsEcpEadunQZdzs8TqnPcyMDXvheU2nPD0iy47V4/TmxpMJzk6DopM0NCJivHkQ3pC7sdzpqCBJtBQuYAWbwbzs0S3WLg80I+Ky4N6DAztIHaPPgYr0LyYpyKnlSifuhPQz758LgqJQtmSN8G1h/eh0MQlLkNpRG6uJYHHN/BXnoe0eZNbB5NZ6rDphdw3m5QTXBBQbF2DBhE5FK9APRBH4d9jHVnuV9R+/Lg7CqF/9rXvMa+trXvkY33HADffKTn6Rt27bRZz/7WXrXu97VO+dP//RPqV6v03vf+16am5ujK664gr7zne9QNps9Ss0Gg2Gtwta9wXBmYdUpe3/nd36Hfud3fmfgccdx6JOf/CR98pOffFENMxgMawe27g2GMwcm0mMwGAwGwzrCmhXp6Q4lFGcTyizK7yZol2pD+EaUB9EMFY7iBf1Dm7SwwdmV2f5tUb4ALoRlYDatTGNwuIUO7em1IZRtKM6zrS7KsW2sU5EGLL8J4YEYZgfP2p6QsTwxRK4kIG7Srqr+Artye5Ttih3InpY/pNN79S8Pem4imQ2rPaKEKsBeiyFSCN3HaP8fZAcmknZKv8B2s3BehY4OCO8L52DMpSlahJv6GJqp+qEL44m+Cn5DnCZt02n/soqEpDhwKHYG20jXMuLAIQocIVZFJMcTx8+FLJhRQY5XBFWIrJpZGcKFmSbjPI8LZn8MZ+UekFTYlpuEKNjDg9EpK/+dMt8Xh6xbVAJOG4f71udBdCj6qRARxePVXrkNvipxTpwmfVBK4MMAe5cWgElBhAbnt5OXiyxXZAeV5iTvPYUMOK50VMgdhMk6Uf+wayL5vI0khM8Hh3G3ITMhhlBjxsF2Wy6eizY/0ys/PsthelFBzsfWKIxtGbK85lFIS7ZHiDmVuD5vXvqcYEZFFH3CNb8iY+jyUBzjsrdf/AaDwWAwrCOsuV/86fOJMZLnxT3itvxugt96YvBkTVrwLVddk6C0LHzjjDvqV1idf6E1IOFO3JVfJWNRH3/Dw6Qpsfpmi8eiFGR9O7LuKOZvpnEbfv1Hyt076v+LP4FfjFGkIgHgmzzWF6tv+DFECURR99d+vlQHHoO6O5JtEYllUEpUfVvHfoig/0X/dDMDr8F+jRLZd3gM50akIh1i8PiP0LO5jUmSZLtTiOwQ7Yl0P4BX/lHqwz6P4et8FMMcVL9aom7cm7NpepTQhzWE5XYujw32HX5OROJXDa6lSK/TDib3gWtU3WJ+Rbg2+6+XpYP8d9LCOQTzpKM0IiJkgHB+y59pOLZRhHsKRNV0U3VN/3WxYv+EH+kJsIZOC/dFccnAPSlpqv6GyZuA5gjuq/oa8Xy4D61YB3yssYB9guvAHXgNzhNsQxJL1gLbGjf670FL9+LnE3tVhO8GJUHcSvuel+q53u4/n0jMBaXVEC1/fmzr3knX2M7wzDPPWPpOg+E44cCBA7R58+ZT3YxfC1v3BsPxw69b92vuxZ8kCR08eJDSNKWtW7fSgQMH1nVWr+VUpuu5H6wPlrCafkjTlBYWFmjTpk3kumvfomfrnmHzfQnWD6vvg2Nd92uO6nddlzZv3txT67J0nkuwfrA+WMax9kOlUvm156wV2LpfCeuDJVg/rK4PjmXdr/2fAgaDwWAwGI4b7MVvMBgMBsM6wpp98YdhSH/2Z39GYRj++pPPYFg/WB8sYz30w3p4xl8H64MlWD+cuD5Yc859BoPBYDAYThzW7C9+g8FgMBgMxx/24jcYDAaDYR3BXvwGg8FgMKwj2IvfYDAYDIZ1BHvxGwwGg8GwjrAmX/w33XQTnXPOOZTNZumyyy6j+++//1Q36YRiz5499JrXvIZKpRKNjY3R29/+dtq3b584p9Vq0e7du2lkZISKxSJdddVVNDU1dYpafOJx4403kuM4dN111/U+Wy998Oyzz9If/uEf0sjICOVyOfqN3/gNeuCBB3rH0zSlj3/847Rx40bK5XK0a9cuevTRR09hi48P1tO6tzXfH7buT9K6T9cYbr/99jQIgvR//I//kf7Lv/xL+sd//MdptVpNp6amTnXTThiuvPLK9NZbb00feeSR9KGHHkr/zb/5N+nWrVvTxcXF3jnve9/70i1btqR79+5NH3jggfS1r31t+rrXve4UtvrE4f7770/POeec9MILL0w/+MEP9j5fD30wMzOTnn322el73vOe9L777kufeOKJ9K677kofe+yx3jk33nhjWqlU0q9//evpT3/60/Tf/tt/m27bti1tNpunsOUvDutt3duaXwlb9ydv3a+5F/+ll16a7t69u/d3HMfppk2b0j179pzCVp1cHDp0KCWi9O67707TNE3n5ubSTCaT3nHHHb1zfvGLX6RElN5zzz2nqpknBAsLC+n27dvT7373u+lv/dZv9TaA9dIHH/nIR9Irrrhi4PEkSdKJiYn0v/yX/9L7bG5uLg3DMP1f/+t/nYwmnhCs93W/ntd8mtq6P9nrfk1R/Z1Ohx588EHatWtX7zPXdWnXrl10zz33nMKWnVzMz88TEdHw8DARET344IPU7XZFv+zYsYO2bt16xvXL7t276S1veYt4VqL10wff+MY36JJLLqHf+73fo7GxMbrooovoi1/8Yu/4k08+SZOTk6IfKpUKXXbZZadtP9i6X99rnsjW/cle92vqxT89PU1xHNP4+Lj4fHx8nCYnJ09Rq04ukiSh6667ji6//HK64IILiIhocnKSgiCgarUqzj3T+uX222+nH//4x7Rnz54Vx9ZLHzzxxBN088030/bt2+muu+6i97///fQnf/In9JWvfIWIqPesZ9IaWe/rfj2veSJb90Qnf92vOVne9Y7du3fTI488Qj/84Q9PdVNOKg4cOEAf/OAH6bvf/S5ls9lT3ZxThiRJ6JJLLqFPf/rTRER00UUX0SOPPEK33HILXX311ae4dYYTgfW65ols3S/jZK/7NfWLf3R0lDzPW+GxOTU1RRMTE6eoVScP1157LX3zm9+kf/7nf6bNmzf3Pp+YmKBOp0Nzc3Pi/DOpXx588EE6dOgQvfrVrybf98n3fbr77rvpc5/7HPm+T+Pj42d8HxARbdy4kV7xileIz8477zzav38/EVHvWc+kNbKe1/16XvNEtu6XcbLX/Zp68QdBQBdffDHt3bu391mSJLR3717auXPnKWzZiUWapnTttdfS1772Nfre975H27ZtE8cvvvhiymQyol/27dtH+/fvP2P65Y1vfCM9/PDD9NBDD/X+XXLJJfSud72rVz7T+4CI6PLLL18R1vWrX/2Kzj77bCIi2rZtG01MTIh+qNVqdN999522/bAe172t+SXYul/CSV/3L8AB8YTi9ttvT8MwTL/85S+nP//5z9P3vve9abVaTScnJ091004Y3v/+96eVSiX9/ve/nz733HO9f41Go3fO+973vnTr1q3p9773vfSBBx5Id+7cme7cufMUtvrEA71703R99MH999+f+r6f/vmf/3n66KOPpl/96lfTfD6f/s//+T9759x4441ptVpN//Ef/zH92c9+lr7tbW87I8L51tO6tzU/GLbuT/y6X3Mv/jRN07/+679Ot27dmgZBkF566aXpvffee6qbdEJBRH3/3Xrrrb1zms1m+h/+w39Ih4aG0nw+n/7u7/5u+txzz526Rp8E6A1gvfTBnXfemV5wwQVpGIbpjh070i984QvieJIk6cc+9rF0fHw8DcMwfeMb35ju27fvFLX2+GE9rXtb84Nh6/7Er3snTdN09TyBwWAwGAyG0xFrysZvMBgMBoPhxMJe/AaDwWAwrCPYi99gMBgMhnUEe/EbDAaDwbCOYC9+g8FgMBjWEezFbzAYDAbDOoK9+A0Gg8FgWEewF7/BYDAYDOsI9uI3GAwGg2EdwV78BoPBYDCsI9iL32AwGAyGdYT/HzZaBjXsx7VQAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Self-Attn matrix shape: (1, 4, 365, 365)\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAF2CAYAAAAFo2PRAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAT4JJREFUeJzt3XtYFNf9P/D3osKisCAqVwEVjXgFg4po45UEL7GS0MRoGpEoxhb9qtiqJFSI2mBMjZBIvMQojUq9RMVGjQbxkp8RTUSpt0iVUPHCosYAgnKRPb8/LFNXdlkGWUTn/XqeeeKeOWfmc2AzH87MmRmVEEKAiIiIqrF40gEQERE1VkySRERERjBJEhERGcEkSUREZASTJBERkRFMkkREREYwSRIRERnBJElERGQEkyQREZERTJJEZlZcXAxHR0ds3LixXrerUqkwbdq0et1mYzZx4kTY2Ng0+H7feOMNvP766w2+X2ocmCTNIDs7G++88w46dOgAtVoNjUaDAQMGICEhAffu3XvS4cm2Z88eqFQquLq6QqfTVVt//fp1xMbGIjMzs9q65ORkxMfHmz/IBlKX/iQkJMDW1hZvvPGGVLZnzx7ExsbWb3CNWE3fkcZu7ty52LZtG/71r3896VDoCWCSrGe7d+9Gjx49sGXLFowePRqffvop4uLi4OHhgT//+c+YMWPGkw5Rto0bN6Jdu3bIy8vDgQMHqq2/fv063n//fSZJAyoqKpCQkIDJkyejSZMmUvmePXvw/vvvmyHCxqmm70hj16tXL/Tu3RtLly590qHQE8AkWY9ycnLwxhtvwNPTE+fPn0dCQgLCw8MRERGBf/zjHzh//jy6detmtL1Op0NpaWkDRmxaSUkJdu7cicjISPTq1aveTxk2FhcuXEBFRUW9b3fXrl24efMmT9c95V5//XVs374dxcXFTzoUamiC6s3UqVMFAPH999/Xqj4AERERITZs2CC6du0qmjZtKnbs2CGEEOLkyZNi+PDhwtbWVrRo0UIMHTpUpKen67UvLy8XsbGxomPHjsLKyko4ODiIAQMGiG+//Vaqk5eXJyZOnCjc3NyEpaWlcHZ2Fr/97W9FTk5OrWJcv369sLCwEHl5eeLDDz8UGo1G3Lt3T1p/8OBBAaDasm7dOjFo0KBq5Z6ennrtNm/eLBYtWiTc3NyElZWVGDp0qLh48WKNMW3dulUAEIcOHaq2buXKlQKAOHPmjMm+FRcXi7Vr14oBAwYIAOLXX3+tsX5N/TFmwoQJol27dnploaGhBn9mD8cVGRkp2rZtKywtLcVzzz0nPvroI6HT6fS2U/X9edjChQuFSqUSn3zyiVS2Z88e8Zvf/EY0b95c2NjYiJEjR4qzZ89Wi6lFixbi6tWrYsyYMaJFixaidevWYvbs2eL+/fs19tGUmr4jVbZs2SKef/55oVarRatWrcSbb74prl69ajDGh506dUq0bt1aDBo0SNy5c0cIIcTVq1dFWFiYcHR0FJaWlqJr167iiy++MBhTbb9///rXvwQAsX379sf6WdDTh0myHrm5uYkOHTrUuj4A0aVLF9GmTRvx/vvvi8TERHHq1Clx9uxZ0aJFC+Hi4iIWLlwoFi9eLNq3by+srKzEsWPHpPbvvvuuUKlUIjw8XHz++edi6dKlYty4cWLx4sVSnf79+ws7OzsRHR0t1qxZIz744AMxZMgQcfjw4VrFOHz4cDFs2DAhhBCXL18WKpVKbNmyRVqv1WrFggULBAAxZcoUsX79erF+/XqRnZ0tvv32W+Hr6ytat24tlVf9EVB1kOrVq5fw8/MTy5YtE7GxsaJ58+aib9++NcZ09+5dYWNjI/74xz9WWzdkyBDRrVu3GtsfO3ZMhIeHC1tbWwFA+Pn5ieXLl5tMBjX1x5iOHTuKV199Va/s6NGj4sUXXxQApO2sX79eCCGETqcTQ4cOFSqVSkyePFksX75cjB49WgAQM2fO1NvOo0nyvffeEyqVSqxevVoq+/LLL4VKpRLDhw8Xn376qfjwww9Fu3bthL29vd4fSqGhoUKtVotu3bqJt99+W6xYsUKEhIQIAOKzzz6rsY8PO3HihAgJCdH7Q6qm74gQQqxbt04AEH369BHLli0T8+bNE9bW1qJdu3Z6f7g8miR/+OEH0bJlS/Hiiy+Ku3fvSvtq27atcHd3FwsWLBArVqwQv/3tbwUAsWzZMqmt3O9fRUWFsLa2FrNnz671z4KeDUyS9aSwsFAAEGPGjKl1GwDCwsJCnDt3Tq88ODhYWFpaSgcRIYS4fv26sLW1FQMHDpTKfHx8xKhRo4xu/9dffxUAxEcffVT7jjwkPz9fNG3aVHz++edSWf/+/av18ccff6w2MqgyatQog6OtqoNUly5dRFlZmVSekJBQq5HguHHjhKOjo15iy8vLExYWFmLBggXV6t+8eVN8/PHHolu3bgKAaN26tZg5c6b417/+VeN+atsfQyoqKoRKpTJ4YI2IiNAbPVZJSUkRAMSiRYv0yn/3u98JlUolLl26JJU9nCRnz54tLCwsRFJSkrT+zp07wt7eXoSHh+ttS6vVCjs7O73yqtHtoz+7qiRSW999951o3ry5GD58uN7v1dh3pLy8XDg6Ooru3bvrJdZdu3YJAGL+/Pl6MVYlySNHjgiNRiNGjRolSktLpTqTJk0SLi4u4tatW3r7eeONN4SdnZ2UTOvy/XvuuefEiBEjav2zoGcDr0nWk6KiIgCAra2trHaDBg1C165dpc+VlZX49ttvERwcjA4dOkjlLi4uGD9+PI4cOSLty97eHufOncPFixcNbtva2hqWlpY4dOgQfv31V7ldwqZNm2BhYYGQkBCpbNy4cfjmm2/qtD1DwsLCYGlpKX1+4YUXAAA///xzje3Gjh2LGzdu4NChQ1LZV199BZ1Oh7Fjx0pl//73v/H666/Dzc0Nf/7zn9GuXTt89dVXuH79OpYtW4aePXvWSz8MuX37NoQQaNmyZa3b7NmzB02aNMH//d//6ZXPnj0bQgh88803euVCCEybNg0JCQnYsGEDQkNDpXWpqakoKCjAuHHjcOvWLWlp0qQJ/P39cfDgwWr7nzp1qt7nF154Qe93UVFRgdLSUqNLnz59sHXrVhw8eBC/+93vTF7nPXHiBG7cuIE//vGPUKvVUvmoUaPg7e2N3bt3V2tz8OBBBAUFYdiwYdi+fTusrKykn8W2bdswevRoCCH0+hwUFITCwkKcPHlSb1tyvn8tW7bErVu3auwPPXuaPukAnhUajQYAcOfOHVnt2rdvr/f55s2buHv3Ljp37lytbpcuXaDT6XDlyhV069YNCxYswJgxY/Dcc8+he/fuGD58ON566y3pwG9lZYUPP/wQs2fPhpOTE/r164eXX34ZEyZMgLOzMwCgsLBQ77YUS0tLODg4AAA2bNiAvn374pdffsEvv/wC4MFMv/LycmzduhVTpkyR1VdDPDw89D5XJRRTSXj48OGws7PD5s2bMWzYMADA5s2b4evri+eee06qd/ToUWzduhUtWrTA6tWr8dZbb8HCoua/De/du4fCwkK9sqqfV10IIWpd9/Lly3B1da32x1aXLl2k9Q/78ssvUVxcjBUrVmDcuHF666r+eBo6dKjBfVV9Z6uo1Wq0adNGr6xly5Z6v4tx48Zh27ZtterL119/jYSEBPzpT38yWqeqP4a+797e3jhy5IheWWlpKUaNGgU/Pz9s2bIFTZv+7xB28+ZNFBQUYPXq1Vi9erXB/d24cUPvs5zvnxACKpXKaF/o2cQkWU80Gg1cXV1x9uxZWe2sra3rvM+BAwciOzsbO3fuxLfffos1a9Zg2bJlWLlyJSZPngwAmDlzJkaPHo2UlBTs27cPf/nLXxAXF4cDBw6gV69emDFjBv7+979L2xw0aBAOHTqEixcv4scffwQAdOrUqdq+N27cWC9J8uHbIh5mKrFYWVkhODgYO3bswGeffYb8/Hx8//33+OCDD/TqjR49GnFxcVi7di0mTpyIv/zlLwgNDcXEiRPh5eVlcNubN29GWFiYrHgMcXBwgEqlqrdRtyEDBgxAZmYmli9fjtdff136AweAdE/r+vXrDSb5hxMMYPx38bBp06bh5ZdfrrHO7du38e6776Jly5b47W9/W5tu1JqVlRVGjhyJnTt3Yu/evXqxVPX397//vd6I+mGPnjmQ8/379ddfDf6/QM82Jsl69PLLL2P16tVIT09HQEBAnbbRpk0bNG/eHFlZWdXWXbhwARYWFnB3d5fKHBwcEBYWhrCwMBQXF2PgwIGIjY2VkiQAeHl5Yfbs2Zg9ezYuXrwIX19fLF26FBs2bMCcOXPw+9//Xqpb9Zf0xo0b0axZM6xfv77ageTIkSP45JNPkJubCw8Pjxr/ujbnX95jx47F3//+d6SlpeGnn36CEELvVCsAtGrVCvPmzcO8efNw+PBhrFmzBkuXLsWiRYswcOBAhIWF4bXXXkOLFi2kNkFBQUhNTX3s/jRt2hReXl7Iycmp9XY8PT2xf/9+3LlzR280eeHCBWn9wzp27IglS5Zg8ODBGD58ONLS0qR2VX8EODo6IjAwsNZx12Tw4ME1ri8oKMCwYcOg0WiQlpYmjepr6i8AZGVlVRvxZmVlVeuvSqXCxo0bMWbMGLz22mv45ptvpJjatGkDW1tbVFZW1lt/q9y/fx9Xrlyp96RPT4EndC30mXTp0iXRokUL0bVrV6HVag2uj4+Plz7DwBR+IR5M3LGystKbfajVaoVGo9GbuPPo5AQhhHjttddE69athRBClJSU6E2GEEKIyspK4eTkJH73u9/V2JeOHTuKoUOHGlx39epVoVKppFm0P/30U7XZg1XGjh0r7O3tq5VXTZzYunWrXnlOTo7RSUCPKi8vFw4ODiIsLEz069fP5KzYKgUFBSIxMVH06tVLABA2NjYiLCxMbwKHMcb6Y8xbb70l3N3dq5XPnTvX4G0nVRN3Pvjgg2r7rWniTnp6urCxsRGDBg2SJqcUFhYKjUYjBg0aJMrLy6vFcOPGDenfhm6vEEKImJgYgxOMjFm7dq1o1apVtQlRxr4jVRN3evbsqTcBZ8+ePTVO3Ll796544YUXhI2NjTh+/LhUZ+LEicLS0tLgxJuH+yv3+1d1C8i2bdtq94OgZwaTZD3buXOnUKvVomXLlmLGjBni888/F4mJieLNN98UlpaWYsqUKVJdY0my6hYQNzc38de//lV8+OGHokOHDtVuAXF0dBSvv/66+PDDD8Xnn38u3nnnHaFSqcT06dOFEA/uIXNwcBBTp04Vn3zyifjss8+kWw+++uoro304duyYAKCX0B/l5+cnevToIYR4cKCzt7cXnTt3FmvWrBH/+Mc/xM8//yyEEGLJkiUCgJg1a5ZITk4W//znP4UQ9ZMkhRBi8uTJwsbGRqhUKrF06dJatXnYyZMnxR//+Edhb29v8j5JIYz3x5ivvvpKABBZWVl65Vu2bBEAxFtvvSU2bNgg/vGPfwghHvwRM2TIEKFSqcSUKVNEYmKiGDNmTK1uAUlLSxNWVlZi5MiRUlLcuHGjsLCwEN27dxeLFi0Sq1atEu+9957w9fXVa1tfSVKIB7OMH1XTd6TqFhB/f38RHx8voqKiRPPmzU3eAlJYWCj8/PyEg4ODlBS1Wq3w9PQUzZs3FzNmzBCrVq0ScXFx4rXXXhMtW7aU2sr9/v3tb38TzZs3F0VFRbJ+FvT0Y5I0g3//+98iPDxctGvXTlhaWgpbW1sxYMAA8emnn+r9tWwsSQrx4OAdFBQkbGxsRPPmzcWQIUPE0aNH9eosWrRI9O3bV9jb2wtra2vh7e0t/vrXv0oHyFu3bomIiAjh7e0tWrRoIezs7IS/v7/efY6GTJ8+XQDQuwXlUbGxsQKANGLYuXOn9ECEhw8yxcXFYvz48cLe3t7gwwQeN0mmpqYKAEKlUokrV67Uqo0h9+7dE5WVlSbrGeuPMWVlZaJ169Zi4cKFeuX3798X06dPF23atBEqlUovEd25c0fMmjVLuLq6imbNmolOnTrV+mECO3fuFE2bNhVjx46V+nPw4EERFBQk7OzshFqtFl5eXmLixInixIkTUrv6TJLGGPuOCCHE5s2bRa9evaSHYtT2YQK3bt0SXbt2Fc7OztJDAPLz80VERIRwd3cXzZo1E87OzmLYsGF694/K/f75+/uL3//+9/XwU6CnjUqIOsxIIKJaW7hwIdatW4eLFy/WanIMNS6ZmZl4/vnncfLkSfj6+j7pcKiBMUkSmVlxcTE6dOiAZcuW4c0333zS4ZBMb7zxBnQ6HbZs2fKkQ6EngEmSiIjICD5xh4iIyAizJcnbt2/jzTffhEajgb29PSZNmmTyNTODBw+GSqXSWx59TBYRET09vvvuO4wePRqurq5QqVRISUmpsX5eXh7Gjx+P5557DhYWFpg5c6bBelu3boW3tzfUajV69OiBPXv26K0XQmD+/PlwcXGBtbU1AgMDjT7CsyZmS5Jvvvkmzp07h9TUVOzatQvfffddrZ7QEh4ejry8PGlZsmSJuUIkIiIzKykpgY+PDxITE2tVv6ysDG3atEF0dDR8fHwM1jl69CjGjRuHSZMm4dSpUwgODkZwcLDeE8+WLFmCTz75BCtXrsTx48fRokULBAUFyX9nrzmmzJ4/f14AED/++KNU9s033wiVSiWuXbtmtN2gQYPEjBkzzBESERE9YQBMvl7uYcZywuuvv17tDUj+/v7inXfeEUI8eOWcs7Oz3huQCgoKhJWVlXRPcm2Z5bF06enpsLe3R+/evaWywMBAWFhY4Pjx43jllVeMtt24cSM2bNgAZ2dnjB49Gn/5y1/QvHlzo/XLyspQVlYmfdbpdLh9+zZatWrFhxET0TNBCIE7d+7A1dXV5AP6a6u0tBTl5eV1jufR46uVlZX0RhZzS09PR2RkpF5ZUFCQdCo3JycHWq1W7/GEdnZ28Pf3R3p6Ot54441a78ssSVKr1cLR0VF/R02bwsHBAVqt1mi78ePHw9PTE66urjh9+jTmzp2LrKwsbN++3WibuLg4vP/++/UWOxFRY3XlyhW0bdv2sbdTWlqK9p420N6orFN7GxubanNMYmJiEBsb+9ix1YZWq4WTk5NemZOTk5Rfqv5bU53akpUk582bhw8//LDGOj/99JOsAB728DXLHj16wMXFBcOGDUN2drbRNzZERUXp/UVRWFgIDw8PtI2NhsVD76cjInpa6UpLcTV2kez31RpTXl4O7Y1K5GR4QmMrb2RadEeH9n6XceXKFb3XrTXUKLKhyUqSs2fPxsSJE2us06FDBzg7O1d7b9v9+/dx+/ZtWe/l8/f3BwBcunTJaJI0NsS3UKuZJInomVLfl5A0thayk6TUVqOp9k7ShuLs7Iz8/Hy9svz8fCm/VP03Pz8fLi4uenXkPjVJVpJs06ZNtZeyGhIQEICCggJkZGTAz88PAHDgwAHodDop8dVGZmYmAOh1koiI6kel0KFS5uNkKoXOPMHIEBAQgLS0NL3bQ1JTU6VXFLZv3x7Ozs5IS0uTkmJRURGOHz+OP/zhD7L2ZZZrkl26dMHw4cMRHh6OlStXoqKiAtOmTcMbb7wBV1dXAMC1a9cwbNgwfPnll+jbty+ys7ORnJyMkSNHolWrVjh9+jRmzZqFgQMHVntRKhERPT4dBHSQlyXl1i8uLsalS5ekzzk5OcjMzISDgwM8PDwQFRWFa9eu4csvv5TqVA2QiouLcfPmTWRmZsLS0hJdu3YFAMyYMQODBg3C0qVLMWrUKGzatAknTpzA6tWrATwYcc+cOROLFi1Cp06d0L59e/zlL3+Bq6srgoODZcVvtpcub9y4EdOmTcOwYcNgYWGBkJAQfPLJJ9L6iooKZGVl4e7duwAAS0tL7N+/H/Hx8SgpKYG7uztCQkIQHR1trhCJiBRNBx3kjgvltjhx4gSGDBkifa6aQxIaGoqkpCTk5eUhNzdXr02vXr2kf2dkZCA5ORmenp74z3/+AwDo378/kpOTER0djXfffRedOnVCSkoKunfvLrWbM2cOSkpKMGXKFBQUFOA3v/kN9u7dC7XMy3DP3LNbi4qKYGdnB4/Fi3hNkoieCbrSUuTOi0ZhYWG9XAesOk5eueBWp4k77t7X6i2Wxs5sI0kiImrcGuJ069OODzgnIiIygiNJIiKF0kGgkiPJGjFJEhEpFE+3msYkSUSkUJVCoFLm3E259Z92TJJERAql++8it42ScOIOERGRERxJEhEpVGUdJu7Irf+0Y5IkIlKoSoE6PLvVPLE0VkySREQKxWuSpjFJEhEplA4qVELe67d0Mus/7ZgkiYgUSiceLHLbKAlntxIRERnBkSQRkUJV1uF0q9z6TzsmSSIihWKSNI1JkohIoXRCBZ2QOXFHZv2nHZMkEZFCcSRpGpMkEZFCVcIClTLnb1aaKZbGirNbiYiIjOBIkohIoUQdrkkKXpMkIiIl4DVJ05gkiYgUqlJYoFLIvCapsCfuMEkSESmUDiroZE5N0fFVWUREpAQ83WoaZ7cSEREZwZEkEZFC1e2aJE+3EhGRAjy4Jsn3SdbE7KdbExMT0a5dO6jVavj7++OHH36osf7WrVvh7e0NtVqNHj16YM+ePeYOkYhIkXT/feKOnEXuRJ/vvvsOo0ePhqurK1QqFVJSUky2OXToEJ5//nlYWVmhY8eOSEpK0lvfrl07qFSqaktERIRUZ/DgwdXWT506VVbsgJmT5ObNmxEZGYmYmBicPHkSPj4+CAoKwo0bNwzWP3r0KMaNG4dJkybh1KlTCA4ORnBwMM6ePWvOMImIFKnqdKvcRY6SkhL4+PggMTGxVvVzcnIwatQoDBkyBJmZmZg5cyYmT56Mffv2SXV+/PFH5OXlSUtqaioA4LXXXtPbVnh4uF69JUuWyIodAFRCmO8Es7+/P/r06YPly5cDAHQ6Hdzd3TF9+nTMmzevWv2xY8eipKQEu3btksr69esHX19frFy5slb7LCoqgp2dHTwWL4KFWl0/HSEieoJ0paXInReNwsJCaDSax95e1XEyObM7mts2kdX27p1KjPc9W6dYVCoVduzYgeDgYKN15s6di927d+sNjt544w0UFBRg7969BtvMnDkTu3btwsWLF6FSPTgdPHjwYPj6+iI+Pl5WjI8y20iyvLwcGRkZCAwM/N/OLCwQGBiI9PR0g23S09P16gNAUFCQ0fpERPRskZsHysvLsWHDBrz99ttSgqyyceNGtG7dGt27d0dUVBTu3r0rOx6zTdy5desWKisr4eTkpFfu5OSECxcuGGyj1WoN1tdqtUb3U1ZWhrKyMulzUVHRY0RNRKQclUKFSpnPYq2q/+ix1srKClZWVo8dk7E8UFRUhHv37sHa2lpvXUpKCgoKCjBx4kS98vHjx8PT0xOurq44ffo05s6di6ysLGzfvl1WPE/97Na4uDi8//77TzoMIqKnTt1elfXgCp27u7teeUxMDGJjY+srtFr74osvMGLECLi6uuqVT5kyRfp3jx494OLigmHDhiE7OxteXl613r7ZkmTr1q3RpEkT5Ofn65Xn5+fD2dnZYBtnZ2dZ9QEgKioKkZGR0ueioqJqvzwiIqpOJyygkzkRR/ffaSxXrlzRuyZZH6NIwHge0Gg01UaRly9fxv79+2s1OvT39wcAXLp0SVaSNNs1SUtLS/j5+SEtLU0q0+l0SEtLQ0BAgME2AQEBevUBIDU11Wh94MEvRqPR6C1ERGSa3Ns/Hh55Pnrcra8kKScPrFu3Do6Ojhg1apTJ7WZmZgIAXFxcZMVj1tOtkZGRCA0NRe/evdG3b1/Ex8ejpKQEYWFhAIAJEybAzc0NcXFxAIAZM2Zg0KBBWLp0KUaNGoVNmzbhxIkTWL16tTnDJCJSJB0g+5qkTuY+iouLcenSJelzTk4OMjMz4eDgAA8PD0RFReHatWv48ssvAQBTp07F8uXLMWfOHLz99ts4cOAAtmzZgt27d+vHodNh3bp1CA0NRdOm+qksOzsbycnJGDlyJFq1aoXTp09j1qxZGDhwIHr27CkrfrMmybFjx+LmzZuYP38+tFotfH19sXfvXumibG5uLiws/jeY7d+/P5KTkxEdHY13330XnTp1QkpKCrp3727OMImIyExOnDiBIUOGSJ+rLo+FhoYiKSkJeXl5yM3Nlda3b98eu3fvxqxZs5CQkIC2bdtizZo1CAoK0tvu/v37kZubi7fffrvaPi0tLbF//35pYObu7o6QkBBER0fLjt+s90k+CbxPkoieNea6T3LFyT6wtpE3VrpXfB9/eP7HeoulsXvqZ7cSEVHd1O0B58p6eRSTJBGRQvEB56YxSRIRKRRHkqYxSRIRKVTdHiagrCSprN4SERHJwJEkEZFC6YQKOrn3Scqs/7RjkiQiUihdHU63yn3p8tOOSZKISKHq9uxWJkkiIlKASqhQKfOWDrn1n3ZMkkRECsWRpGnK6i0REZEMHEkSESlUJeSfPq00TyiNFpMkEZFC8XSraUySREQKxcfSmcYkSUSkUKIODzgXnN1KRERKwJGkacrqLRERkQwcSRIRKRSf3WoakyQRkULxVVmmMUkSESkUR5KmMUkSESmUDhay3+rBt4AQEZEiVAoVKmWODOXWf9op608CIiIiGTiSJCJSKF6TNI1JkohIoUQdnt0qFPYwASZJIiKF4kuXTWOSJCJSKJ2Qf/pUJ8wUTCPFJElEpFB8VZZpZu9tYmIi2rVrB7VaDX9/f/zwww9G6yYlJUGlUuktarXa3CESEZGZfPfddxg9ejRcXV2hUqmQkpJiss2hQ4fw/PPPw8rKCh07dkRSUpLe+tjY2Gq5wtvbW69OaWkpIiIi0KpVK9jY2CAkJAT5+fmy4zdrkty8eTMiIyMRExODkydPwsfHB0FBQbhx44bRNhqNBnl5edJy+fJlc4ZIRKRYuv++KkvuIkdJSQl8fHyQmJhYq/o5OTkYNWoUhgwZgszMTMycOROTJ0/Gvn379Op169ZNL1ccOXJEb/2sWbPw9ddfY+vWrTh8+DCuX7+OV199VVbsgJlPt3788ccIDw9HWFgYAGDlypXYvXs31q5di3nz5hlso1Kp4OzsbM6wiIgIDfMwgREjRmDEiBG1rr9y5Uq0b98eS5cuBQB06dIFR44cwbJlyxAUFCTVa9q0qdFcUVhYiC+++ALJyckYOnQoAGDdunXo0qULjh07hn79+tU6HrONJMvLy5GRkYHAwMD/7czCAoGBgUhPTzfarri4GJ6ennB3d8eYMWNw7ty5GvdTVlaGoqIivYWIiEyruiYpdwFQ7bhbVlZWLzGlp6fr5Q0ACAoKqpY3Ll68CFdXV3To0AFvvvkmcnNzpXUZGRmoqKjQ2463tzc8PDxqzD+GmC1J3rp1C5WVlXByctIrd3JyglarNdimc+fOWLt2LXbu3IkNGzZAp9Ohf//+uHr1qtH9xMXFwc7OTlrc3d3rtR9ERM8qHVTSAwVqvfz3dKu7u7vesTcuLq5eYtJqtQbzRlFREe7duwcA8Pf3R1JSEvbu3YsVK1YgJycHL7zwAu7cuSNtw9LSEvb29tW2Yyz/GNOoZrcGBAQgICBA+ty/f3906dIFq1atwsKFCw22iYqKQmRkpPS5qKiIiZKIqBZEHa4xiv/Wv3LlCjQajVRuZWVVr7HV5OHTtz179oS/vz88PT2xZcsWTJo0qV73ZbYk2bp1azRp0qTabKL8/PxaX3Ns1qwZevXqhUuXLhmtY2Vl1aC/HCIiejDJ8uEkWV+cnZ0N5g2NRgNra2uDbezt7fHcc89JucLZ2Rnl5eUoKCjQG03KyT9VzHa61dLSEn5+fkhLS5PKdDod0tLS9EaLNamsrMSZM2fg4uJirjCJiBRL9qnWOjzrVa6AgAC9vAEAqampNeaN4uJiZGdnS7nCz88PzZo109tOVlYWcnNza51/qpj1dGtkZCRCQ0PRu3dv9O3bF/Hx8SgpKZFmu06YMAFubm7SuewFCxagX79+6NixIwoKCvDRRx/h8uXLmDx5sjnDJCJSpIZ4mEBxcbHe2cCcnBxkZmbCwcEBHh4eiIqKwrVr1/Dll18CAKZOnYrly5djzpw5ePvtt3HgwAFs2bIFu3fvlrbxpz/9CaNHj4anpyeuX7+OmJgYNGnSBOPGjQMA2NnZYdKkSYiMjISDgwM0Gg2mT5+OgIAAWTNbATMnybFjx+LmzZuYP38+tFotfH19sXfvXumibG5uLiws/vcD//XXXxEeHg6tVouWLVvCz88PR48eRdeuXc0ZJhGRIjXEW0BOnDiBIUOGSJ+r5pCEhoYiKSkJeXl5ejNT27dvj927d2PWrFlISEhA27ZtsWbNGr3bP65evYpx48bhl19+QZs2bfCb3/wGx44dQ5s2baQ6y5Ytg4WFBUJCQlBWVoagoCB89tlnsmIHAJUQ4pl6El9RURHs7OzgsXgRLPi0HiJ6BuhKS5E7LxqFhYX1ch2w6jg5+ttJaNbCUlbbipJyfP3SF/UWS2PXqGa3EhFRw+H7JE1T1pNqiYiIZOBIkohIoTiSNI1JkohIoZgkTWOSJCJSKCZJ05gkiYgUSgB1eCydsnDiDhERkREcSRIRKRRPt5rGJElEpFBMkqYxSRIRKRSTpGlMkkRECsUkaRqTJBGRQgmhgpCZ9OTWf9pxdisREZERHEkSESmUDirZ90nKrf+0Y5IkIlIoXpM0jUmSiEiheE3SNCZJIiKF4kjSNCZJIiKF4kjSNM5uJSIiMoIjSSIihRJ1ON2qtJEkkyQRkUIJAELmu6+U9qosJkkiIoXSQQUV75OsEZMkEZFCceKOaUySREQKpRMqqHgLSI04u5WIiMgIjiSJiBRKiDpM3FHYzB0mSSIiheI1SdPMerr1u+++w+jRo+Hq6gqVSoWUlBSTbQ4dOoTnn38eVlZW6NixI5KSkswZIhGRYlUlSbmLHObIA3FxcejTpw9sbW3h6OiI4OBgZGVl6dUZPHgwVCqV3jJ16lRZsQNmTpIlJSXw8fFBYmJirern5ORg1KhRGDJkCDIzMzFz5kxMnjwZ+/btM2eYRESKVPXsVrmLHObIA4cPH0ZERASOHTuG1NRUVFRU4KWXXkJJSYnetsLDw5GXlyctS5YskRU7YObTrSNGjMCIESNqXX/lypVo3749li5dCgDo0qULjhw5gmXLliEoKMhcYRIRKVJDXJM0Rx7Yu3evXpukpCQ4OjoiIyMDAwcOlMqbN28OZ2dneQE/olHNbk1PT0dgYKBeWVBQENLT059QRERE1JDqkgcKCwsBAA4ODnrlGzduROvWrdG9e3dERUXh7t27suNpVBN3tFotnJyc9MqcnJxQVFSEe/fuwdraulqbsrIylJWVSZ+LiorMHicR0bPgwUhS7sSdB/999FhrZWUFKyurx45Jbh7Q6XSYOXMmBgwYgO7du0vl48ePh6enJ1xdXXH69GnMnTsXWVlZ2L59u6x4GlWSrIu4uDi8//77TzoMIqKnzuPMbnV3d9crj4mJQWxsbH2FVmsRERE4e/Ysjhw5olc+ZcoU6d89evSAi4sLhg0bhuzsbHh5edV6+40qSTo7OyM/P1+vLD8/HxqNxuAoEgCioqIQGRkpfS4qKqr2yyMiouoE5D+wvKr+lStXoNFopPL6GEUC8vLAtGnTsGvXLnz33Xdo27Ztjdv19/cHAFy6dOnpTZIBAQHYs2ePXllqaioCAgKMtqmvIT4RkdI8zkhSo9HoJcn6Ups8IITA9OnTsWPHDhw6dAjt27c3ud3MzEwAgIuLi6x4zDpxp7i4GJmZmVJwOTk5yMzMRG5uLoAHo8AJEyZI9adOnYqff/4Zc+bMwYULF/DZZ59hy5YtmDVrljnDJCJSJlHHRQZz5IGIiAhs2LABycnJsLW1hVarhVarxb179wAA2dnZWLhwITIyMvCf//wH//znPzFhwgQMHDgQPXv2lBW/WUeSJ06cwJAhQ6TPVadFQ0NDkZSUhLy8POkHBQDt27fH7t27MWvWLCQkJKBt27ZYs2YNb/8gInpKmSMPrFixAsCDBwY8bN26dZg4cSIsLS2xf/9+xMfHo6SkBO7u7ggJCUF0dLTs+FVCPFtP4isqKoKdnR08Fi+ChVr9pMMhInpsutJS5M6LRmFhYb2c4qw6TnZIeg8WzeUdJ3V3S/HzxL/WWyyNXaO6JklERA2HDzg3jUmSiEih+IBz05gkiYiUSqgeLHLbKAiTJBGRQvF0q2mN6tmtREREjQlHkkRESvU4j9xRCCZJIiKF4sQd05gkiYiUTGEjQ7mYJImIFIojSdOYJImIlIrXJE3i7FYiIiIjOJIkIlIs1X8XuW2Ug0mSiEipeLrVJCZJIiKlYpI0iUmSiEip+OxWk5gkiYgUis9uNY2zW4mIiIzgSJKISKl4TdIkJkkiIqXiNUmTmCSJiBRKJR4sctsoCZMkEZFS8XSrSUySRERKxdOtJnF2KxERkREcSRIRKRVPt5rEJElEpFRMkiYxSRIRKRWTpElMkkRESsWJOyYxSRIRKRTvkzTNrLNbv/vuO4wePRqurq5QqVRISUmpsf6hQ4egUqmqLVqt1pxhEhGRmcjNA8CDXPD888/DysoKHTt2RFJSUrU6iYmJaNeuHdRqNfz9/fHDDz/orS8tLUVERARatWoFGxsbhISEID8/X3b8Zk2SJSUl8PHxQWJioqx2WVlZyMvLkxZHR0czRUhEpGCijosMcvNATk4ORo0ahSFDhiAzMxMzZ87E5MmTsW/fPqnO5s2bERkZiZiYGJw8eRI+Pj4ICgrCjRs3pDqzZs3C119/ja1bt+Lw4cO4fv06Xn31VXnBw8ynW0eMGIERI0bIbufo6Ah7e/v6D4iIiBqU3DywcuVKtG/fHkuXLgUAdOnSBUeOHMGyZcsQFBQEAPj4448RHh6OsLAwqc3u3buxdu1azJs3D4WFhfjiiy+QnJyMoUOHAgDWrVuHLl264NixY+jXr1+t42mUDxPw9fWFi4sLXnzxRXz//fc11i0rK0NRUZHeQkREpqnwv+uStV7+2/bR425ZWVm9xJSeno7AwEC9sqCgIKSnpwMAysvLkZGRoVfHwsICgYGBUp2MjAxUVFTo1fH29oaHh4dUp7YaVZJ0cXHBypUrsW3bNmzbtg3u7u4YPHgwTp48abRNXFwc7OzspMXd3b0BIyYieopVzW6VuwBwd3fXO/bGxcXVS0harRZOTk56ZU5OTigqKsK9e/dw69YtVFZWGqxTNX9Fq9XC0tKy2hnJh+vUVqOa3dq5c2d07txZ+ty/f39kZ2dj2bJlWL9+vcE2UVFRiIyMlD4XFRUxURIR1cZj3Cd55coVaDQaqdjKyqrewmpMGlWSNKRv3744cuSI0fVWVlbP7C+HiKix0mg0ekmyvjg7O1ebhZqfnw+NRgNra2s0adIETZo0MVjH2dlZ2kZ5eTkKCgr0RpMP16mtRnW61ZDMzEy4uLg86TCIiJ49DTC7Va6AgACkpaXplaWmpiIgIAAAYGlpCT8/P706Op0OaWlpUh0/Pz80a9ZMr05WVhZyc3OlOrVl1pFkcXExLl26JH3OyclBZmYmHBwc4OHhgaioKFy7dg1ffvklACA+Ph7t27dHt27dUFpaijVr1uDAgQP49ttvzRkmEZEiNcTDBOTmgalTp2L58uWYM2cO3n77bRw4cABbtmzB7t27pW1ERkYiNDQUvXv3Rt++fREfH4+SkhJptqudnR0mTZqEyMhIODg4QKPRYPr06QgICJA1sxUwc5I8ceIEhgwZIn2uunYYGhqKpKQk5OXlITc3V1pfXl6O2bNn49q1a2jevDl69uyJ/fv3622DiIjqSQM8u1VuHmjfvj12796NWbNmISEhAW3btsWaNWuk2z8AYOzYsbh58ybmz58PrVYLX19f7N27V28yz7Jly2BhYYGQkBCUlZUhKCgIn332mczOAiohxDP1kKGioiLY2dnBY/EiWKjVTzocIqLHpistRe68aBQWFtbLdcCq42S7hX+VfZzUlZbiP395r95iaewa/cQdIiIyDz671bRGP3GHiIjoSeFIkohIqfiqLJOYJImIlIovXTaJSZKISKF4TdI0JkkiIqXiSNIkTtwhIiIygiNJIiKlqsPpVqWNJJkkiYiUiqdbTWKSJCJSKiZJk5gkiYgUirNbTePEHSIiIiOYJImIiIzg6VYiIqXiNUmTmCSJiBSK1yRNY5IkIlIyhSU9uZgkiYiUiqdbTWKSJCJSKJ5uNY2zW4mIiIzgSJKISKl4utUkJkkiIoXi6VbTmCSJiJSKI0mTmCSJiJSKSdIkJkkiIoXi6VbTOLuViIjICI4kiYiUiqdbTWKSJCJSKiZJk8x6ujUuLg59+vSBra0tHB0dERwcjKysLJPttm7dCm9vb6jVavTo0QN79uwxZ5hERIpUdU1S7lIXiYmJaNeuHdRqNfz9/fHDDz8YrVtRUYEFCxbAy8sLarUaPj4+2Lt3r16ddu3aQaVSVVsiIiKkOoMHD662furUqbLiNmuSPHz4MCIiInDs2DGkpqaioqICL730EkpKSoy2OXr0KMaNG4dJkybh1KlTCA4ORnBwMM6ePWvOUImIlEfUcZFp8+bNiIyMRExMDE6ePAkfHx8EBQXhxo0bButHR0dj1apV+PTTT3H+/HlMnToVr7zyCk6dOiXV+fHHH5GXlyctqampAIDXXntNb1vh4eF69ZYsWSIrdpUQosEGzzdv3oSjoyMOHz6MgQMHGqwzduxYlJSUYNeuXVJZv3794Ovri5UrV5rcR1FREezs7OCxeBEs1Op6i52I6EnRlZYid140CgsLodFoHnt7VcfJLtM+QBMrecfJyrJS/LT8XVmx+Pv7o0+fPli+fDkAQKfTwd3dHdOnT8e8efOq1Xd1dcV7772nNyoMCQmBtbU1NmzYYHAfM2fOxK5du3Dx4kWoVCoAD0aSvr6+iI+Pl9XHhzXo7NbCwkIAgIODg9E66enpCAwM1CsLCgpCenq6WWMjIqL6V15ejoyMDL3juoWFBQIDA40e18vKyqB+ZJBjbW2NI0eOGN3Hhg0b8Pbbb0sJssrGjRvRunVrdO/eHVFRUbh7966s+Bts4o5Op8PMmTMxYMAAdO/e3Wg9rVYLJycnvTInJydotVqD9cvKylBWViZ9Lioqqp+AiYiedY8xcefRY62VlRWsrKyqVb916xYqKysNHtcvXLhgcBdBQUH4+OOPMXDgQHh5eSEtLQ3bt29HZWWlwfopKSkoKCjAxIkT9crHjx8PT09PuLq64vTp05g7dy6ysrKwffv2Wna2AZNkREQEzp49a/QvgbqKi4vD+++/X6/bJCJShMdIku7u7nrFMTExiI2NrY+okJCQgPDwcHh7e0OlUsHLywthYWFYu3atwfpffPEFRowYAVdXV73yKVOmSP/u0aMHXFxcMGzYMGRnZ8PLy6tWsTTI6dZp06Zh165dOHjwINq2bVtjXWdnZ+Tn5+uV5efnw9nZ2WD9qKgoFBYWSsuVK1fqLW4iomeZqo4LAFy5ckXv2BsVFWVwH61bt0aTJk1kHdfbtGmDlJQUlJSU4PLly7hw4QJsbGzQoUOHanUvX76M/fv3Y/LkySb76+/vDwC4dOmSybpVzJokhRCYNm0aduzYgQMHDqB9+/Ym2wQEBCAtLU2vLDU1FQEBAQbrW1lZQaPR6C1ERFQLjzG79dHjrqFTrQBgaWkJPz8/veO6TqdDWlqa0eN6FbVaDTc3N9y/fx/btm3DmDFjqtVZt24dHB0dMWrUKJPdzczMBAC4uLiYrFvFrKdbIyIikJycjJ07d8LW1la6rmhnZwdra2sAwIQJE+Dm5oa4uDgAwIwZMzBo0CAsXboUo0aNwqZNm3DixAmsXr3anKESESlOQz27NTIyEqGhoejduzf69u2L+Ph4lJSUICwsDED1PHD8+HFcu3YNvr6+uHbtGmJjY6HT6TBnzhy97ep0Oqxbtw6hoaFo2lQ/nWVnZyM5ORkjR45Eq1atcPr0acyaNQsDBw5Ez549ax27WZPkihUrADyYhvuwdevWSRdYc3NzYWHxvwFt//79kZycjOjoaLz77rvo1KkTUlJSapzsQ0REjdfYsWNx8+ZNzJ8/H1qtFr6+vti7d680mefRPFBaWoro6Gj8/PPPsLGxwciRI7F+/XrY29vrbXf//v3Izc3F22+/XW2flpaW2L9/v5SQ3d3dERISgujoaFmxN+h9kg2B90kS0bPGXPdJdnunbvdJnlsl7z7Jpxmf3UpEpGTP1DCp/jFJEhEpFN8naRqTJBGRUvEtICYxSRIRKRRHkqY16LNbiYiIniYcSRIRKRVPt5rEJElEpFA83WoakyQRkVJxJGkSkyQRkVIxSZrEJElEpFA83WoaZ7cSEREZwZEkEZFS8XSrSUySREQKpRICKpnvuJBb/2nHJElEpFQcSZrEJElEpFCcuGMakyQRkVJxJGkSZ7cSEREZwZEkEZFC8XSraUySRERKxdOtJjFJEhEpFEeSpjFJEhEpFUeSJjFJEhEpmNJGhnJxdisREZERHEkSESmVEA8WuW0UhEmSiEihOHHHNCZJIiKl4sQdk5gkiYgUSqV7sMhtoyRMkkRESsWRpElmnd0aFxeHPn36wNbWFo6OjggODkZWVlaNbZKSkqBSqfQWtVptzjCJiMjMEhMT0a5dO6jVavj7++OHH34wWreiogILFiyAl5cX1Go1fHx8sHfvXr06sbGx1XKFt7e3Xp3S0lJERESgVatWsLGxQUhICPLz82XFbdYkefjwYURERODYsWNITU1FRUUFXnrpJZSUlNTYTqPRIC8vT1ouX75szjCJiBSpauKO3EWuzZs3IzIyEjExMTh58iR8fHwQFBSEGzduGKwfHR2NVatW4dNPP8X58+cxdepUvPLKKzh16pRevW7duunliiNHjuitnzVrFr7++mts3boVhw8fxvXr1/Hqq6/Kit2sp1sfzfxJSUlwdHRERkYGBg4caLSdSqWCs7OzOUMjIqIGugXk448/Rnh4OMLCwgAAK1euxO7du7F27VrMmzevWv3169fjvffew8iRIwEAf/jDH7B//34sXboUGzZskOo1bdrUaK4oLCzEF198geTkZAwdOhQAsG7dOnTp0gXHjh1Dv379ahV7gz5MoLCwEADg4OBQY73i4mJ4enrC3d0dY8aMwblz54zWLSsrQ1FRkd5CRESmPc5I8tHjbllZmcF9lJeXIyMjA4GBgVKZhYUFAgMDkZ6ebrBNWVlZtcts1tbW1UaKFy9ehKurKzp06IA333wTubm50rqMjAxUVFTo7dfb2xseHh5G92tIgyVJnU6HmTNnYsCAAejevbvRep07d8batWuxc+dObNiwATqdDv3798fVq1cN1o+Li4OdnZ20uLu7m6sLRETPFlHHBYC7u7vesTcuLs7gLm7duoXKyko4OTnplTs5OUGr1RpsExQUhI8//hgXL16ETqdDamoqtm/fjry8PKmOv78/kpKSsHfvXqxYsQI5OTl44YUXcOfOHQCAVquFpaUl7O3ta71fQxpsdmtERATOnj1b7S+BRwUEBCAgIED63L9/f3Tp0gWrVq3CwoULq9WPiopCZGSk9LmoqIiJkoioFh7nYQJXrlyBRqORyq2srOotroSEBISHh8Pb2xsqlQpeXl4ICwvD2rVrpTojRoyQ/t2zZ0/4+/vD09MTW7ZswaRJk+otlgYZSU6bNg27du3CwYMH0bZtW1ltmzVrhl69euHSpUsG11tZWUGj0egtRERkXo8ed40lydatW6NJkybVZpXm5+cbvZ7Ypk0bpKSkoKSkBJcvX8aFCxdgY2ODDh06GI3H3t4ezz33nJQrnJ2dUV5ejoKCglrv1xCzJkkhBKZNm4YdO3bgwIEDaN++vextVFZW4syZM3BxcTFDhEREClY1cUfuIoOlpSX8/PyQlpYmlel0OqSlpemdNTRErVbDzc0N9+/fx7Zt2zBmzBijdYuLi5GdnS3lCj8/PzRr1kxvv1lZWcjNzTW534eZ9XRrREQEkpOTsXPnTtja2krnge3s7GBtbQ0AmDBhAtzc3KTz2QsWLEC/fv3QsWNHFBQU4KOPPsLly5cxefJkc4ZKRKQ4DfXs1sjISISGhqJ3797o27cv4uPjUVJSIs12fTQPHD9+HNeuXYOvry+uXbuG2NhY6HQ6zJkzR9rmn/70J4wePRqenp64fv06YmJi0KRJE4wbNw7AgzwzadIkREZGwsHBARqNBtOnT0dAQECtZ7YCZk6SK1asAAAMHjxYr3zdunWYOHEiACA3NxcWFv8b0P76668IDw+HVqtFy5Yt4efnh6NHj6Jr167mDJWISHka6Ik7Y8eOxc2bNzF//nxotVr4+vpi79690mSeR/NAaWkpoqOj8fPPP8PGxgYjR47E+vXr9SbhXL16FePGjcMvv/yCNm3a4De/+Q2OHTuGNm3aSHWWLVsGCwsLhISEoKysDEFBQfjss89kxa4S4tl670lRURHs7OzgsXgRLPikHiJ6BuhKS5E7LxqFhYX1Mu+i6jjZP2gBmjaTd5y8X1GKo/vm11ssjR2f3UpEpFQ68WCR20ZBGvRhAkRERE8TjiSJiJSKbwExiUmSiEihVKjD7FazRNJ4MUkSESlVAz3g/GnGJElEpFANdZ/k04wTd4iIiIzgSJKISKk4ccckJkkiIoVSCQGVzGuMcus/7ZgkiYiUSvffRW4bBWGSJCJSKI4kTWOSJCJSKl6TNImzW4mIiIzgSJKISKn4MAGTmCSJiBSKDxMwjUmSiEipOJI0iUmSiEihVLoHi9w2SsIkSUSkVBxJmsTZrUREREZwJElEpFS8T9IkJkkiIoXiE3dMY5IkIlIqXpM0iUmSiEipBOQ/sFxZOZJJkohIqXi61TTObiUiIjKCI0kiIqUSqMM1SbNE0mgxSRIRKRUn7phk1tOtK1asQM+ePaHRaKDRaBAQEIBvvvmmxjZbt26Ft7c31Go1evTogT179pgzRCIi5dLVcamDxMREtGvXDmq1Gv7+/vjhhx+M1q2oqMCCBQvg5eUFtVoNHx8f7N27V69OXFwc+vTpA1tbWzg6OiI4OBhZWVl6dQYPHgyVSqW3TJ06VVbcZk2Sbdu2xeLFi5GRkYETJ05g6NChGDNmDM6dO2ew/tGjRzFu3DhMmjQJp06dQnBwMIKDg3H27FlzhklEpEhVE3fkLnJt3rwZkZGRiImJwcmTJ+Hj44OgoCDcuHHDYP3o6GisWrUKn376Kc6fP4+pU6filVdewalTp6Q6hw8fRkREBI4dO4bU1FRUVFTgpZdeQklJid62wsPDkZeXJy1LliyR+zNq2LGzg4MDPvroI0yaNKnaurFjx6KkpAS7du2Syvr16wdfX1+sXLmyVtsvKiqCnZ0dPBYvgoVaXW9xExE9KbrSUuTOi0ZhYSE0Gs1jb6/qODms25/RtImVrLb3K8uQdu4jWbH4+/ujT58+WL58OQBAp9PB3d0d06dPx7x586rVd3V1xXvvvYeIiAipLCQkBNbW1tiwYYPBfdy8eROOjo44fPgwBg4cCODBSNLX1xfx8fGy+viwBpvdWllZiU2bNqGkpAQBAQEG66SnpyMwMFCvLCgoCOnp6Q0RIhER1bPy8nJkZGToHdstLCwQGBho9NheVlYG9SODHGtraxw5csTofgoLCwE8GIg9bOPGjWjdujW6d++OqKgo3L17V1b8Zp+4c+bMGQQEBKC0tBQ2NjbYsWMHunbtarCuVquFk5OTXpmTkxO0Wq3R7ZeVlaGsrEz6XFRUVD+BExE96x5j4s6jx1orKytYWVUfld66dQuVlZUGj+0XLlwwuIugoCB8/PHHGDhwILy8vJCWlobt27ejsrLSYH2dToeZM2diwIAB6N69u1Q+fvx4eHp6wtXVFadPn8bcuXORlZWF7du317q7Zk+SnTt3RmZmJgoLC/HVV18hNDQUhw8fNpoo5YqLi8P7779fL9siIlKUx0iS7u7uesUxMTGIjY2tl7ASEhIQHh4Ob29vqFQqeHl5ISwsDGvXrjVYPyIiAmfPnq020pwyZYr07x49esDFxQXDhg1DdnY2vLy8ahWL2U+3WlpaomPHjvDz80NcXBx8fHyQkJBgsK6zszPy8/P1yvLz8+Hs7Gx0+1FRUSgsLJSWK1eu1Gv8RETPrMeY3XrlyhW9Y29UVJTBXbRu3RpNmjSRdWxv06YNUlJSUFJSgsuXL+PChQuwsbFBhw4dqtWdNm0adu3ahYMHD6Jt27Y1dtff3x8AcOnSpRrrPazBn7ij0+n0To8+LCAgAGlpaXplqampRq9hAg+G+FW3mFQtRERk2uPMbn30uGvoVCvwYKDk5+end2zX6XRIS0ur8dgOAGq1Gm5ubrh//z62bduGMWPGSOuEEJg2bRp27NiBAwcOoH379ib7m5mZCQBwcXExWbeKWU+3RkVFYcSIEfDw8MCdO3eQnJyMQ4cOYd++fQCACRMmwM3NDXFxcQCAGTNmYNCgQVi6dClGjRqFTZs24cSJE1i9erU5wyQiUqYGephAZGQkQkND0bt3b/Tt2xfx8fEoKSlBWFgYgOq54Pjx47h27Rp8fX1x7do1xMbGQqfTYc6cOdI2IyIikJycjJ07d8LW1laau2JnZwdra2tkZ2cjOTkZI0eORKtWrXD69GnMmjULAwcORM+ePWsdu1mT5I0bNzBhwgTk5eXBzs4OPXv2xL59+/Diiy8CAHJzc2Fh8b/BbP/+/ZGcnIzo6Gi8++676NSpE1JSUvQuxBIR0dNl7NixuHnzJubPnw+tVgtfX1/s3btXmszzaC4oLS1FdHQ0fv75Z9jY2GDkyJFYv3497O3tpTorVqwA8OA2j4etW7cOEydOhKWlJfbv3y8lZHd3d4SEhCA6OlpW7A1+n6S58T5JInrWmOs+yUCvmXW6T3J/dny9xdLY8dmtRERKxWe3msQkSUSkWHVIkgp7DQiTJBGRUnEkaRKTJBGRUukEZI8MdcpKkg1+nyQREdHTgiNJIiKlEroHi9w2CsIkSUSkVLwmaRKTJBGRUvGapElMkkRESsWRpElMkkRESiVQhyRplkgaLc5uJSIiMoIjSSIipeLpVpOYJImIlEr30FuUZbVRDiZJIiKl4kjSJCZJIiKlYpI0iUmSiEipeJ+kSZzdSkREZARHkkRECiWEDkLms1jl1n/aMUkSESmVEPJPn/KaJBERKYKowzVJJkkiIlIEnQ5Q8VVZNWGSJCJSKo4kTeLsViIiIiM4kiQiUiih00HIPN3K2a1ERKQMPN1qEpMkEZFS6QSgYpKsCZMkEZFSCQHZbwFhkiQiIiUQOgEhcyQpFJYkzTq7dcWKFejZsyc0Gg00Gg0CAgLwzTffGK2flJQElUqlt6jVanOGSEREDSAxMRHt2rWDWq2Gv78/fvjhB6N1KyoqsGDBAnh5eUGtVsPHxwd79+6Vvc3S0lJERESgVatWsLGxQUhICPLz82XFbdYk2bZtWyxevBgZGRk4ceIEhg4dijFjxuDcuXNG22g0GuTl5UnL5cuXzRkiEZFyCV3dFpk2b96MyMhIxMTE4OTJk/Dx8UFQUBBu3LhhsH50dDRWrVqFTz/9FOfPn8fUqVPxyiuv4NSpU7K2OWvWLHz99dfYunUrDh8+jOvXr+PVV1+VFbtKNPDY2cHBAR999BEmTZpUbV1SUhJmzpyJgoKCOm+/qKgIdnZ28Fi8CBYchRLRM0BXWorcedEoLCyERqN57O1VHScHq15BU1UzWW3viwocEjtkxeLv748+ffpg+fLlAACdTgd3d3dMnz4d8+bNq1bf1dUV7733HiIiIqSykJAQWFtbY8OGDbXaZmFhIdq0aYPk5GT87ne/AwBcuHABXbp0QXp6Ovr161er2BvsmmRlZSW2bt2KkpISBAQEGK1XXFwMT09P6HQ6PP/88/jggw/QrVs3o/XLyspQVlYmfS4sLATw4EtFRPQsqDqe1feY5r4okz0yvI8KAA8S7cOsrKxgZWVVrX55eTkyMjIQFRUllVlYWCAwMBDp6ekG91FWVlbtUpu1tTWOHDlS621mZGSgoqICgYGBUh1vb294eHjISpIQZnb69GnRokUL0aRJE2FnZyd2795ttO7Ro0fF3//+d3Hq1Clx6NAh8fLLLwuNRiOuXLlitE1MTEzVjT5cuHDh8kwvNR0L5bh3755wdnaucxw2NjbVymJiYgzu69q1awKAOHr0qF75n//8Z9G3b1+DbcaNGye6du0q/v3vf4vKykrx7bffCmtra2FpaVnrbW7cuFGq/7A+ffqIOXPm1PpnZfaRZOfOnZGZmYnCwkJ89dVXCA0NxeHDh9G1a9dqdQMCAvRGmf3790eXLl2watUqLFy40OD2o6KiEBkZKX3W6XS4ffs2WrVqBZVKVf8dqkFRURHc3d1x5cqVejkl8rRgv9lvJXiS/RZC4M6dO3B1da2X7anVauTk5KC8vLzO8Tx6fDU0iqyrhIQEhIeHw9vbGyqVCl5eXggLC8PatWvrbR+1ZfYkaWlpiY4dOwIA/Pz88OOPPyIhIQGrVq0y2bZZs2bo1asXLl26ZLSOoSG+vb39Y8X8uKpm8yoN+60s7HfDsrOzq9ftqdXqBrl7oHXr1mjSpEm1WaX5+flwdnY22KZNmzZISUlBaWkpfvnlF7i6umLevHno0KFDrbfp7OyM8vJyFBQU6OWEmvZrSIM/4Fyn0+ldQ6xJZWUlzpw5AxcXFzNHRURE5mBpaQk/Pz+kpaVJZTqdDmlpaTXOTwEeJHI3Nzfcv38f27Ztw5gxY2q9TT8/PzRr1kyvTlZWFnJzc03u92FmHUlGRUVhxIgR8PDwwJ07d5CcnIxDhw5h3759AIAJEybAzc0NcXFxAIAFCxagX79+6NixIwoKCvDRRx/h8uXLmDx5sjnDJCIiM4qMjERoaCh69+6Nvn37Ij4+HiUlJQgLCwNQPRccP34c165dg6+vL65du4bY2FjodDrMmTOn1tu0s7PDpEmTEBkZCQcHB2g0GkyfPh0BAQG1n7QDMyfJGzduYMKECcjLy4OdnR169uyJffv24cUXXwQA5ObmwsLif4PZX3/9FeHh4dBqtWjZsiX8/Pxw9OhRg9cvGyMrKyvExMTU67n5pwH7zX4rgVL7XR/Gjh2LmzdvYv78+dBqtfD19cXevXvh5OQEoHouKC0tRXR0NH7++WfY2Nhg5MiRWL9+vd5pU1PbBIBly5bBwsICISEhKCsrQ1BQED777DNZsTf4fZJERERPC750mYiIyAgmSSIiIiOYJImIiIxgkiQiIjKCSfIx3b59G2+++SY0Gg3s7e0xadIkFBcX16qtEAIjRoyASqVCSkqKeQOtZ3L7ffv2bUyfPh2dO3eGtbU1PDw88H//93/Ss3YbKzmv9wGArVu3wtvbG2q1Gj169MCePXsaKNL6Jaffn3/+OV544QW0bNkSLVu2RGBgoMmfU2Ml9/ddZdOmTVCpVAgODjZvgNTwav0AOzJo+PDhwsfHRxw7dkz8v//3/0THjh3FuHHjatX2448/FiNGjBAAxI4dO8wbaD2T2+8zZ86IV199Vfzzn/8Uly5dEmlpaaJTp04iJCSkAaOWZ9OmTcLS0lKsXbtWnDt3ToSHhwt7e3uRn59vsP73338vmjRpIpYsWSLOnz8voqOjRbNmzcSZM2caOPLHI7ff48ePF4mJieLUqVPip59+EhMnThR2dnbi6tWrDRz545Hb7yo5OTnCzc1NvPDCC2LMmDENEyw1GCbJx3D+/HkBQPz4449S2TfffCNUKpW4du1ajW1PnTol3NzcRF5e3lOXJB+n3w/bsmWLsLS0FBUVFeYI87H17dtXRERESJ8rKyuFq6uriIuLM1j/9ddfF6NGjdIr8/f3F++8845Z46xvcvv9qPv37wtbW1vx97//3VwhmkVd+n3//n3Rv39/sWbNGhEaGsok+Qzi6dbHkJ6eDnt7e/Tu3VsqCwwMhIWFBY4fP2603d27dzF+/HgkJibKeoZgY1HXfj+q6n10TZs22Bvbaq3qVTwPv2bH1Ot90tPT9eoDQFBQkNH6jVFd+v2ou3fvoqKiAg4ODuYKs97Vtd8LFiyAo6Ojwffj0rOh8R2dniJarRaOjo56ZU2bNoWDgwO0Wq3RdrNmzUL//v2l5xA+bera74fdunULCxcuxJQpU8wR4mO7desWKisr9Z7eAQBOTk64cOGCwTZardZg/dr+TBqDuvT7UXPnzoWrq2u1Pxgas7r0+8iRI/jiiy+QmZnZABHSk8KRpAHz5s2DSqWqcantAeNR//znP3HgwAHEx8fXb9D1wJz9flhRURFGjRqFrl27IjY29vEDp0Zj8eLF2LRpE3bs2NEgb5h4Uu7cuYO33noLn3/+OVq3bv2kwyEz4kjSgNmzZ2PixIk11unQoQOcnZ1x48YNvfL79+/j9u3bRk+jHjhwANnZ2dVe5xUSEoIXXngBhw4deozIH485+13lzp07GD58OGxtbbFjxw40a9bsccM2i7q83sfZ2VlW/caoLv2u8re//Q2LFy/G/v370bNnT3OGWe/k9js7Oxv/+c9/MHr0aKlMp9MBeHBWJSsrC15eXuYNmhrGk74o+jSrmsBy4sQJqWzfvn01TmDJy8sTZ86c0VsAiISEBPHzzz83VOiPpS79FkKIwsJC0a9fPzFo0CBRUlLSEKE+lr59+4pp06ZJnysrK4Wbm1uNE3defvllvbKAgICncuKOnH4LIcSHH34oNBqNSE9Pb4gQzUJOv+/du1ft/+MxY8aIoUOHijNnzoiysrKGDJ3MiEnyMQ0fPlz06tVLHD9+XBw5ckR06tRJ71aIq1evis6dO4vjx48b3QaestmtQsjvd2FhofD39xc9evQQly5dEnl5edJy//79J9WNGm3atElYWVmJpKQkcf78eTFlyhRhb28vtFqtEEKIt956S8ybN0+q//3334umTZuKv/3tb+Knn34SMTExT+0tIHL6vXjxYmFpaSm++uorvd/rnTt3nlQX6kRuvx/F2a3PJibJx/TLL7+IcePGCRsbG6HRaERYWJjewSEnJ0cAEAcPHjS6jacxScrt98GDBwUAg0tOTs6T6UQtfPrpp8LDw0NYWlqKvn37imPHjknrBg0aJEJDQ/Xqb9myRTz33HPC0tJSdOvWTezevbuBI64fcvrt6elp8PcaExPT8IE/Jrm/74cxST6b+KosIiIiIzi7lYiIyAgmSSIiIiOYJImIiIxgkiQiIjKCSZKIiMgIJkkiIiIjmCSJiIiMYJIkIiIygkmSiIjICCZJIiIiI5gkiYiIjGCSJCIiMuL/A5JXWF/0LD7uAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhAAAAHOCAYAAADJ3DBLAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASp5JREFUeJzt3XlYVGX/P/D3AM6wMwgii7KIEC75SJDmkppiZpaappY+T0qpaVRqWmmbyzfFpZ5S89HUIvetMq3MJVGzXEhDkzQjFTETN8ANBGU+vz/4ccbjsN0movJ+Xddc1+HMfc7c52Y+M+85c84cg4gIiIiIiBTYVXYHiIiI6M7DAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQXSHmTx5MiIiImCxWG7aOseMGQODwYAzZ87ctHXeDtq0aYM2bdqU2W7z5s0wGAzYvHlzhffpTnHy5Ek8+eST8PLygsFgwIcffnhXjtPatWvh6uqK06dPV3ZX7jgMEHeAQ4cO4fnnn0edOnXg6OgId3d3tGjRAlOnTkVubm5ld0/ZmjVrYDAY4O/vX+yb4N9//40xY8Zgz549NvctXrwYH3744U3v07Zt2zBmzBhkZ2ff9HXfTOfPn8ekSZPw+uuvw86usHxzcnIwZsyYu+pFnSrfsGHDsG7dOowaNQoLFizAI488ctMf40afu4cOHULv3r3h4+MDJycnhIWF4c033yyx/ZUrV1C/fn0YDAa89957uvseeeQR1K1bF/Hx8TeyCVWaQ2V3gEr37bffokePHjCZTHjmmWfQsGFD5Ofn48cff8Srr76K3377DbNnz67sbipZtGgRgoODkZaWhsTERMTExOju//vvvzF27FgEBwejcePGuvsWL16MlJQUDB069Kb2adu2bRg7diz69esHs9l8U9d9M3366ae4evUqnn76aW1eTk4Oxo4dCwDl+rRdlaxfv76yu3DHSkxMRJcuXTBixAhtXnh4OHJzc2E0Gm/KY9zIc3fPnj1o06YNAgICMHz4cHh5eSE9PR3Hjh0rcZnp06cjPT29xPuff/55jBgxAmPHjoWbm5vSNlRlDBC3sSNHjuCpp55CUFAQEhMT4efnp90XFxeHP//8E99++22Jy1ssFuTn58PR0fFWdLdcLl26hFWrViE+Ph4JCQlYtGiRTYC4ExUUFODgwYOoX79+hT5OQkICOnfufFv9T29nN+uNrio6deqUTZi2s7Mr13MvJycHzs7ON71PFosF//nPfxAREYFNmzbBycmpzGVOnTqFcePG4fXXX8c777xTbJvu3bvjpZdewooVK/Dss8/e7G7fvYRuW4MGDRIA8tNPP5WrPQCJi4uThQsXSv369cXBwUFWrlwpIiK//PKLPPLII+Lm5iYuLi7Stm1b2b59u275/Px8GTNmjNStW1dMJpNUr15dWrRoIevXr9fanDhxQvr16ycBAQFiNBrF19dXOnfuLEeOHClXHxcsWCB2dnZy4sQJmTRpkri7u0tubq52/6ZNmwSAzS0hIUFat25tMz8oKEi33LJly+Tdd9+VgIAAMZlM0rZtW0lNTS21T6NHjy72McuzTampqTJq1Cjx8/OTLl26lNp2ypQpAkDS0tJs7hs5cqRUq1ZNMjMzS1z+8OHDAkA+++wzbd6RI0eK7fvo0aO1Nhs3bpSWLVuKs7OzeHh4SOfOnWX//v3FjsHp06e1eWlpaRIaGioNGjSQjIwMERHJysqSIUOGSK1atcRoNEpoaKhMnDhRCgoKbPo0ZcoU+fjjj6VOnTpiNBolOjpakpKSSh2j8oiLixMXFxe5dOmSzX1PPfWU1KxZU65evSoiIq1bt5bWrVvr2hw7dky6dOkizs7OUqNGDRk6dKisXbtWAMimTZt0bXfs2CEdOnQQd3d3cXJyklatWsmPP/5o87jlqa/yWL9+vbRo0UI8PDzExcVFwsPDZdSoUbo2ly9flnfeeUdCQ0PFaDRKrVq15NVXX5XLly/btBs6dKh4e3uLq6urPP7443Ls2DGb58f1EhISin1OiVjr7Npxat26tTRo0EB27dolDz74oDg5OcmQIUNEROTnn3+Whx9+WLy8vMTR0VGCg4MlNjZWRMr33L3ed999JwBkzZo1IiJy6dIl7X9dktjYWGnSpIlWP1OmTCm2XWRkpHTu3LnUdZEeA8RtLCAgQOrUqVPu9gCkXr16UqNGDRk7dqzMmDFDkpOTJSUlRVxcXMTPz0/+7//+TyZOnCghISFiMplkx44d2vJvvPGGGAwGGTBggMyZM0fef/99efrpp2XixIlam+bNm4uHh4e89dZbMnfuXJkwYYI89NBDsmXLlnL18ZFHHpF27dqJiMjRo0fFYDDI8uXLtfszMjJk3LhxAkAGDhwoCxYskAULFsihQ4dk/fr10rhxY/H29tbmFwWkohe2yMhIiYqKkg8++EDGjBkjzs7O0qRJk1L7tHfvXnn66acFgHzwwQfaui9evFhs+5ycHFmwYIEWaEwmk/Tq1avMoFe0vZMnT7a5r06dOtKpU6dSl1+4cKEAkF9//VWbd/HiRZk5c6YAkCeeeELr+969e0VEZMOGDeLg4CDh4eEyefJkGTt2rHh7e4unp6cuIF0fIP78808JDAyUxo0ba/MuXbokjRo1Ei8vL3njjTdk1qxZ8swzz4jBYNDeMESsbwyRkZFSt25dmTRpkkyePFm8vb2lVq1akp+fb7Ntw4YNkwULFpS6/UV++OEHAaB73hT1z8XFReLi4rR51weInJwcCQ8PF0dHR3nttdfkww8/lKioKGnUqJHNG+PGjRvFaDRKs2bN5P3335cPPvhAGjVqJEajUXbu3Km1K299lSUlJUULWlOnTpVZs2bJiBEjpFWrVlqbgoICefjhh8XZ2VmGDh0qH3/8sbz44ovi4OBgE2D//e9/CwDp3bu3fPTRR9KtWzdtO0t7kz506JAsWLBAAEj79u2155RIyQHC19dXatSoIS+99JJ8/PHH8tVXX8nJkyfF09NTwsPDZcqUKTJnzhx58803pV69eiJS9nO3OMOHDxcAsnHjRomKihIAYjQapVevXnL27Fmb9jt37hQ7OzvZtm2bLtgWp3///uLt7V3iY5MtBojb1Llz5wRAmZ9qrwVA7Ozs5LffftPN79q1qxiNRjl06JA27++//xY3Nzfdi9O//vWvUt/EsrKySi3Aspw8eVIcHBxkzpw52rzmzZvbbOPPP/+s7XW4XqdOnbS9DtcqemGrV6+e5OXlafOnTp0qAGTfvn2l9q1o70Bpex127dolgwcPFg8PDwEgUVFR8tFHH5W61+B6zZo1k6ioKN28pKQkASDz588vddm33npLAMiFCxd080+fPl3im0Ljxo3Fx8dH9+K6d+9esbOzk2eeeUabd22AOHDggPj7+8v999+v27b/+7//ExcXF/njjz90jzFy5Eixt7eX9PR0EbEGCC8vL93yq1atEgDy9ddf65YfMWKEAJCPPvqo1O0vYrFYJCAgQLp3766bv3z5cgEgP/zwgzbv+gDx4Ycf2oSPS5cuSd26dXVvjBaLRcLCwqRDhw5isVi0tjk5ORISEiLt27fX5pW3vsrywQcf2OwFul7RHrytW7fq5s+aNUu3t3LPnj0CQF544QVdu969e5cZIIoU7dG8VkkBAoDMmjVL13blypUCQH7++ecSH6O0525xOnfurD23+vTpI59//rm8/fbb4uDgIM2bN9f9rywWizRp0kSefvppEZEyA8SECRMEgJw8ebJcfSERnoVxmzp//jwAKB/Q07p1a9338AUFBVi/fj26du2KOnXqaPP9/PzQu3dv/Pjjj9pjmc1m/Pbbb0hNTS123U5OTjAajdi8eTOysrJUNwlLly6FnZ0dunfvrs17+umn8d13393Q+ooTGxur+977wQcfBAAcPnz4hte5ePFiREZGIjo6GitWrEBsbCz27t2LXbt2IS4uDp6enuVeV69evbB7924cOnRIm7ds2TKYTCZ06dKl1GXPnj0LBwcHuLq6luuxTpw4gT179qBfv36oXr26Nr9Ro0Zo37491qxZY7NMSkoKWrdujeDgYHz//fe6bVuxYgUefPBBeHp64syZM9otJiYGBQUF+OGHH2y29drli/tfvPXWW3jvvfcwYcIEPPfcc7h8+XKJt7y8PACAwWBAjx49sGbNGly8eFFb17JlyxAQEICWLVuWOCZr1qyBn58fnnzySW2es7MzBg4cqGu3Z88epKamonfv3jh79qy2rZcuXUK7du3www8/wGKxKNVXWYqON1i1alWJp+iuWLEC9erVQ0REhO5/0LZtWwDApk2btO0EgJdfflm3/M0++LiIyWRCbGysbl7R9nzzzTe4cuXKTXmcov/3/fffj4ULF6J79+4YN24c/u///g/btm3Dxo0btbafffYZ9u3bh0mTJpVr3UXP1bvtVOaKxABxm3J3dwcAXLhwQWm5kJAQ3d+nT59GTk4O7rnnHpu29erVg8Vi0Y5eHjduHLKzsxEeHo57770Xr776Kn799VetvclkwqRJk/Ddd9+hZs2aaNWqFSZPnoyMjAytzblz55CRkaHdMjMztfsWLlyIJk2a4OzZs/jzzz/x559/IjIyEvn5+VixYoXSdpYkMDBQ93fRi8I/CSizZ8/Gnj17cN9992Hr1q344IMP0KhRo1KXyczM1I3DuXPnAAA9evSAnZ0dli1bBgAQEaxYsQIdO3bU/uc3y9GjRwGgxP990RvitR5//HG4ublh3bp1Nv1JTU3F2rVrUaNGDd2t6CDYU6dO6dqX9b/YtGkTxo8fDwB444034OTkVOrt2u3o1asXcnNzsXr1agCFbyxr1qxBjx49YDAYSh2TunXr2rS5foyKQnTfvn1ttnfu3LnIy8vDuXPnlOqrLL169UKLFi3Qv39/1KxZE0899RSWL1+uCxOpqan47bffbPoUHh4OwPo/OHr0KOzs7BAaGlrqdt4sAQEBNgestm7dGt27d8fYsWPh7e2NLl26ICEhQQuCN6LooMlrz0ICgN69ewMoPJsKKPwANmrUKLz66quoXbt2udYtIgBQ6vOH9HgWxm3K3d0d/v7+SElJUVquPEcll6RVq1Y4dOgQVq1ahfXr12Pu3Ln44IMPMGvWLPTv3x9A4SeYxx9/HF999RXWrVuHt99+G/Hx8UhMTERkZCSGDBmCefPmaets3bo1Nm/ejNTUVPz8888AgLCwMJvHXrRokc2nwBthb29f7PyiF4cb8d5772HmzJlYvnw56tevj9atWyM2Nhbdu3eHi4tLsct069YNW7Zs0f7u27cvPvvsM/j7++PBBx/E8uXL8cYbb2DHjh1IT08v16ckLy8vXL16FRcuXKiwU826d++OefPmYdGiRXj++ed191ksFrRv3x6vvfZascsWvYkVKet/cd999yE6Ohq7du3CsGHDygxl1+55eeCBBxAcHIzly5ejd+/e+Prrr5Gbm4tevXqVuY3lUfSmPWXKFJtTia/tzz95M7yek5MTfvjhB2zatAnffvst1q5di2XLlqFt27ZYv3497O3tYbFYcO+99+K///1vseso75vlzVbc647BYMDnn3+OHTt24Ouvv8a6devw7LPP4v3338eOHTvKvSftWv7+/gCAmjVr6ub7+PgAsIbT9957D/n5+ejVqxfS0tIAAH/99ZfWJi0tDf7+/rrQU7Sst7e3cr+qKgaI29hjjz2G2bNnY/v27WjWrNkNraNGjRpwdnbGwYMHbe77/fffYWdnp3vRqV69OmJjYxEbG4uLFy+iVatWGDNmjBYgACA0NBTDhw/H8OHDkZqaisaNG+P999/HwoUL8dprr+Hf//631rboU+eiRYtQrVo1LFiwwOaN5ccff8S0adOQnp6OwMDAUj8BVNSng9LWGx0djU8++QRTp07F0qVLMXfuXPTt2xcvvvgievbsidjYWLRo0UK3zPvvv6/b61H0wgcUftJ84YUXcPDgQSxbtgzOzs54/PHHy+xjREQEgMLTe699sy2p70FBQQBQ4v/e29vbJgBNmTIFDg4OeOGFF+Dm5qZ9sgMK/+8XL168aafdenh4YP369Wjbti0WLlyIzZs3K50G27NnT0ydOhXnz5/HsmXLEBwcjAceeKDUZYKCgpCSkgIR0Y3b9WNU9Mnd3d291O1Vra+y2NnZoV27dmjXrh3++9//YsKECXjzzTexadMmxMTEIDQ0FHv37kW7du1Kfc4GBQXBYrHg0KFDur0OxfWzoj3wwAN44IEHMH78eCxevBh9+vTB0qVL0b9/f+V6joqKwpw5c3D8+HHd/L///htA4f8DANLT05GVlYUGDRrYrGPChAmYMGECkpOTdeHwyJEj8Pb21tZBZeNXGLex1157DS4uLujfvz9Onjxpc/+hQ4cwderUUtdhb2+Phx9+GKtWrdKSOFD4M7WLFy9Gy5YttV3VZ8+e1S3r6uqKunXrap+ycnJycPnyZV2b0NBQuLm5aW3q16+PmJgY7RYVFQWgMEA8+OCD6NWrF5588knd7dVXXwUALFmyBAC0N7XifhXSxcVF+zrgZirtMYu4urqif//+2LFjB1JSUvDcc8/hq6++QsuWLREeHo758+drbaOionTjcO0bY/fu3WFvb48lS5ZgxYoVeOyxx0rck3GtohC5a9cu3fyi8+2v77ufnx8aN26MefPm6e5LSUnB+vXr8eijj9o8hsFgwOzZs/Hkk0+ib9++2lcEQOEb9vbt27Fu3Tqb5bKzs3H16tUyt+F6np6e2LBhA2rWrInp06crLdurVy/k5eVh3rx5WLt2LXr27FnmMo8++ij+/vtvfP7559q8nJwcmx9ji4qKQmhoKN577z3dcRZFin72WKW+ynLt131Fit7giuqrZ8+eOH78OObMmWPTNjc3V/tKqmPHjgCAadOm6dpUxK+4liQrK8tmz9/121PSc7ckXbp0gclkQkJCgu6rnblz5wIA2rdvD6Dw2I+VK1fqbh9//DEAoF+/fli5cqXN1727d+++4Q9qVVZlHsFJZVu1apU4OjqKp6enDBkyRObMmSMzZsyQPn36iNFolIEDB2ptUcxR0yLW08wCAgJk/PjxMmnSJKlTp47NaWY+Pj7Ss2dPmTRpksyZM0eef/55MRgM8tJLL4mISHJyslSvXl0GDRok06ZNk//973/Svn17ASCff/55iduwY8cOASAffvhhiW2ioqLk3nvvFZHC36Mwm81yzz33yNy5c2XJkiVy+PBhERGZPHmyAJBhw4bJ4sWLZfXq1SJiPTp8xYoVuvUWHXld3Bkd1yo6E+LRRx+V+fPny5IlS0o8jfNaeXl5snTpUmnfvr1069atzPZFYmJixM3NTQDIF198Ue7lGjZsqB1Vfq369euLr6+vzJgxQ5YsWaKddVJ0GmdERIRMmTJFxo0bJzVq1BBPT09tTEVsT+PMz8+XRx99VEwmk2zcuFFECs9WuO+++8TBwUH69+8vM2fOlPfee0/69u0rLi4u2rKlHe2OEo64P3PmTLGnd5albt262jju3r3b5v7rz8IoOuPC0dFRXn/99VJP49y0aZM4OjpKYGCgjB49WmbPni2jR4+WVq1ayWOPPaa1K299lWXIkCESGRkpb731lsyZM0fGjx8vAQEBUqtWLcnOzhaRwtM4H330UTEYDPLUU0/J9OnT5cMPP5RBgwZJ9erVdWc8FJ2a3KdPH5kxY0a5T+MsUtzrSWm/A3G9Dz74QMLCwuS1116Tjz/+WN577z255557xN3dXffcK+m5W5Ki07zbt28vM2bMkIEDB4rBYCi2Lq5V2vPy5MmTYm9vL3Pnzi11HaTHAHEH+OOPP2TAgAESHBwsRqNR3NzcpEWLFjJ9+nTdj8eUFCBECn/opkOHDuLq6irOzs7y0EMPybZt23Rt3n33XWnSpImYzWZxcnKSiIgIGT9+vPbCfubMGYmLi5OIiAhxcXERDw8Padq0qc35+Nd76aWXBIDuNLfrjRkzRgBo54CvWrVK+zGsawPAxYsXpXfv3mI2m4v9IakbDRAihacpBgQEiJ2dXbl/SOpa5QkcRebMmSMAxM3NTfdDWmX573//K66urpKTk6Obv23bNomKihKj0WjzBvH9999LixYtxMnJSdzd3eXxxx8v1w9J5eTkSOvWrcXV1VV7I7xw4YKMGjVK6tatK0ajUby9vaV58+by3nvvac+TGwkQN+rNN98UAFK3bt1i7y/uh6SOHj0qnTt3FmdnZ/H29pYhQ4aU+ENSycnJ0q1bN/Hy8hKTySRBQUHSs2dPLVQVKU99lWXjxo3SpUsX8ff3F6PRKP7+/vL000/bnDabn58vkyZNkgYNGojJZBJPT0+JioqSsWPHyrlz57R2ubm58vLLL4uXl5e4uLiU+4ekivzTAPHLL7/I008/LYGBgWIymcTHx0cee+wx2bVrl65dac/d4lgsFpk+fbqEh4dLtWrVpHbt2vLWW2+VGUBLe17OnDlTnJ2d5fz586Wug/QMIv/g6DIiuqXOnTuHOnXqYPLkyXjuuecquzt0BzIYDBg9ejTGjBlT2V25bURGRqJNmzb44IMPKrsrdxQeA0F0B/Hw8MBrr72GKVOm3NTLeRNVVWvXrkVqaipGjRpV2V2543APBBFRBbv2t1KK4+TkBA8Pj1vSF+6BoJuFp3ESEVWwa6+kW5yi3wkhupMwQBARVbANGzaUev+1vxNS0bjTmW4WfoVBREREyngQJRERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCDNmDFjYDAYcObMmcruiqZfv34IDg6u7G4Q3bVY93SjGCDornHgwAE88sgjcHV1RfXq1fGf//wHp0+fruxuEVEFSUpKwgsvvICoqChUq1YNBoOhsrtUpTBA0F3hr7/+QqtWrfDnn39iwoQJGDFiBL799lu0b98e+fn5ld09IqoAa9aswdy5c2EwGFCnTp3K7k6VwwBBd4UJEybg0qVLSExMxMsvv4w33ngDy5cvx969e/HZZ59VdveIqAIMHjwY586dw65du9C+ffvK7k6VwwBBNrKzs9GvXz+YzWZ4eHggNjYWOTk5Nu0WLlyIqKgoODk5oXr16njqqadw7NgxXZutW7eiR48eCAwMhMlkQu3atTFs2DDk5ubarO+rr75Cw4YN4ejoiIYNG2LlypXl7vMXX3yBxx57DIGBgdq8mJgYhIeHY/ny5QpbT1Q13Yl1X7NmTTg5OalvLN0UDpXdAbr99OzZEyEhIYiPj8cvv/yCuXPnwsfHB5MmTdLajB8/Hm+//TZ69uyJ/v374/Tp05g+fTpatWqF5ORkmM1mAMCKFSuQk5ODwYMHw8vLC0lJSZg+fTr++usvrFixQlvf+vXr0b17d9SvXx/x8fE4e/YsYmNjUatWrTL7e/z4cZw6dQrR0dE29zVp0gRr1qz554NCdJe70+qebgNC9P+NHj1aAMizzz6rm//EE0+Il5eX9ndaWprY29vL+PHjde327dsnDg4Ouvk5OTk2jxMfHy8Gg0GOHj2qzWvcuLH4+flJdna2Nm/9+vUCQIKCgkrt988//ywAZP78+Tb3vfrqqwJALl++XOo6iKqqO7XurxcXFyd8S7u1+BUG2Rg0aJDu7wcffBBnz57F+fPnAQBffvklLBYLevbsiTNnzmg3X19fhIWFYdOmTdqy1+5evHTpEs6cOYPmzZtDRJCcnAwAOHHiBPbs2YO+ffvCw8NDa9++fXvUr1+/zP4W7RY1mUw29zk6OuraEFHx7rS6p8rHrzDIxrXHEQCAp6cnACArKwvu7u5ITU2FiCAsLKzY5atVq6ZNp6en45133sHq1auRlZWla3fu3DkAwNGjRwGg2PXdc889+OWXX0rtb9GLVV5ens19ly9f1rUhouLdaXVPlY8BgmzY29sXO19EAAAWiwUGgwHfffddsW1dXV0BAAUFBWjfvj0yMzPx+uuvIyIiAi4uLjh+/Dj69esHi8VyU/rr5+cHoPATzfVOnDiB6tWrF7t3gois7rS6p8rHAEHKQkNDISIICQlBeHh4ie327duHP/74A/PmzcMzzzyjzd+wYYOuXVBQEAAgNTXVZh0HDx4ssz8BAQGoUaMGdu3aZXNfUlISGjduXOY6iKh0t1vdU+XjMRCkrFu3brC3t8fYsWO1TydFRARnz54FYP1Ec20bEcHUqVN1y/j5+aFx48aYN2+etnsTKHzB2b9/f7n61L17d3zzzTe608k2btyIP/74Az169FDbQCKycTvWPVUu7oEgZaGhoXj33XcxatQopKWloWvXrnBzc8ORI0ewcuVKDBw4ECNGjEBERARCQ0MxYsQIHD9+HO7u7vjiiy9svhMFgPj4eHTq1AktW7bEs88+i8zMTEyfPh0NGjTAxYsXy+zTG2+8gRUrVuChhx7CkCFDcPHiRUyZMgX33nsvYmNjK2IYiKqU27Hujx49igULFgCAtgfy3XffBVC4h+M///nPTRwBsnHrT/yg21XR6VynT5/WzU9ISBAAcuTIEd38L774Qlq2bCkuLi7i4uIiEREREhcXJwcPHtTa7N+/X2JiYsTV1VW8vb1lwIABsnfvXgEgCQkJNuurV6+emEwmqV+/vnz55ZfSt2/fcp/OlZKSIg8//LA4OzuL2WyWPn36SEZGxo0MBVGVcSfX/aZNmwRAsbfWrVvf4IhQeRlErtsXRURERFQGHgNBREREyhggiIiISBkDBBERESmrsACRmZmJPn36wN3dHWazGc8991yZR9W2adMGBoNBd7v+51WJ6PbFuieqOiosQPTp0we//fYbNmzYgGeffRbz58+Hh4cHmjZtiqSkpBKXGzBgAGbPno3Q0FCYTCZs3bqVV1MkukMU1f3zzz8PJycnJCQkIDg4uNSaB4B27dppNR8REYGYmJhb1GMiulEVEiAOHDiAtWvXYu7cuUhLS8OMGTPw0ksvwWKxICwsDB06dMCpU6eKXTY7OxuDBw/G888/j+TkZHTr1g1du3ZFSkpKRXSViG6Sorrv2bMnpk2bhgkTJmDmzJk4e/YsHn744RJr/ty5c0hMTNRq/sknn0Tv3r1Z80S3uQo5jfPTTz/F8OHDkZWVhaZNm8LR0RFHjx7F0aNHUbduXWRnZ2P48OEYOXKkbrk2bdrgxx9/REFBgb6TBgMGDhyIWbNm2TxWXl6e7iJKFosFmZmZ8PLygsFguNmbRlSliAguXLgAf39/2NmV/nmjqO7Dw8O1ms/IyEBeXh7c3d0xatQom5oHAHd3d1y4cEE3r7SaB1j3RBVFpeYr5Iekxo8fL+Hh4ZKXlycGg0EcHBzk008/FU9PT2nRooUYjUbp0KGDzXIff/yxuLm5idFolI8++kh8fX2lY8eO8sorr0ijRo2KfayiH0HhjTfeKu527NixctV9WFiYruZ/++03cXR0FHt7+2JrXkTE2dlZjEajJCYmanVft27dEmuedc8bbxV/K0/NK+2BGDlyJCZNmlRqmwMHDuDLL7/EvHnzsGnTJgQEBKB79+74/PPP4ePjg9GjR2PkyJEwm8266xYUcXBwgMlkwqVLl5CYmIh27dphzJgx+N///oeTJ0/atD9//rx2vfqivxs0aIDWYS/Cwd6Eo128dO0v187XphuHpWvTL/tt1LVbltlUm/5+WyNt2vWINZGda3RFt0zEu39p038MC7Ju0wX9JyLzn9Yhd1/2szZ9ZPz9unbVAq0Hnxl/ctema36SrGt37JVIFMdO3z1U339Vmy4wWft0toH1ynoujc/qlsndZR2/6gese4b+fkS/l+iet45o02mD7tGmnU/qn16ux619sAyyPtbVhT66dgWO1v4Zrnmo515dpWuXkNZcmzbN9NSmr7ha/09Z4forB14OsA5Mw3Dr/yx1S4iu3X+esD4nPlvfVpuu9a+/de3+3uWvTXeIsV7Q65uf79O1c/zb2g/PVOtGWfqd0bXL3GMdi7VPT9Om266N06b3dfpMt8x9S57Tpv0aW69K+t/Qz3Xtnpv+sjbtnGHtQ+w7+nGd+EvHwr7l5uGvlyejLEV1/8knn+Dw4cNazQNAjRo1cOHCBdSoUaPMmgeg1b2XlxfOnDlj0x4oue7v7fE27Ks54nwdfXu3o9bpq07W59alAP3z01A7R5u+km29gmv1vdb/nf11V40/d811pYJXWq/p8Pcb+na+T1svEGX4Vz1tOtffWdfu727W52dwgnV+2nW/yG53wlGbNmZZt6lRp9917Q6siLA+7lXr9ua2tr6+OOxy06/bWqbI8bcu49tA/xqct6KmNt11SKI2veRQtK6d71TrVRMuvGod47PZrrp21UzXvGjts77mWRyv+z/Vtfbd2dH6mu4227odBXGZumUGB23RpuN/f0SbHhWxVtduYcYD2vSJVdbX8VxvfR8CH7C+dhRMqaFNZ4Xpr/zrkGudNl60XonUPFhfC78fs47l3jaLtOmobU9p055uObplTh22vj6Li7We+zTeqWsX6ZymTY+Za/1577z7LunaOTvnoyAnD78/Ow3Z2dnw8PBAaZSuhTF8+HD069ev1DZ16tSBr68vTp06hfz8wn9sdHQ0rl69iszMTPj7+yMoKAjHjx8vcR2XL19GUFCQ9lXG77//XmLbyZMnY/z48SqbQUSKkpKS4ObmVuL9RXV/+vRpAIU1DwBXr15FVlYW/P39bb6muFZRzVssFvzrX/8CAJuvMq/Fuie6DZS5j+IG7N+/XwDI6tWrBYDEx8fLunXrxGAwyPHjx6VBgwZiNpuLXdbHx0d69+4tycnJMn36dAEgRqNRIiIiim3/5ptvVvquHt54u9tv5dmdWVT3QGHNi4hW9+Hh4eWq+c2bN0vz5s0FgISEhJT4WKx73nir2NtN/wpDRceOHfHXX38hJSUFbdq0wfHjxxEdHY2FCxfCw8MDbm5ucHd3x/z589GkSRMcOnQIixcvxtatW1FQUIAhQ4Zg2LBh8Pf3x88//4wGDRpg9+7dNo9T0sFU1apVQ2BgII4dOwZ3d3eb5aqK8+fPo3bt2hwHjgMA9XEQlQOqADz00EPYvHkz2rZti3HjxiE2NhZRUVH45ptvYDabsWPHDrRr105X9127doWnpyfmz5+PX3/9FUOHDkVaWhoiIyOLrXmAdV8WPt8LcRwKqYyDSs1X2OW8Fy1ahMGDByMlJQWbN29Gy5Yt8corr2Dw4MHIy8vDfffdhx9++AFvvfUWoqKi8MILL+D7779HcnIyLly4gL1796JLly7w8fHBTz/9hJo1axb7OCaTCSaT/jsns9msfT/q7u5epZ84RTgOhTgOhVTGoazvQa+1ZMkS+Pn5ITExEdu2bUPHjh3h5OSEvLw8NGjQAFeuXMHBgwcxdepULFq0CEajEfb29ti6dSvq1q2LgIAA1KpVC2lpaSXWPMC6Ly+OQyGOQ6HyjkN5a77CfkiqevXqWLZsGZo0aYJWrVohPT0dLVq0QHJyMsxmMzp27AgRQX5+Pk6cOIHatWtjy5YtOH/+PJYvXw4vLy8sXLgQq1evRkBAACIiIsp+UCKqVL6+vlrN+/j44Ntvv0VKSgrMZjPatGmD4OBgtG7dGtWqVQMA1K5dG3v27MHy5csRGhqKjIwMZGdns+aJ7gAVfi2MV155BTt37sS4ceOwZ88eREZG4sqVK4iNLTykODAwEH5+flr7cePGwcPDA9999x22b9+ORo0a4cyZM+jfv39Fd5WIbgLWPFHVUGFfYRTp1asXTp8+jXfeeQcZGRlo3Lgx1q5dq+2eTE9P133PkpWVhQEDBiAjIwOenp6IiorCtm3bUL9+faXHNZlMGD16tM1uzqqG41CI41DoVoxDZdU8wP9zEY5DIY5DoYoahwo7iJKIiIjuXrycNxERESljgCAiIiJlDBBERESkjAGCiIiIlN2VAWLGjBkIDg6Go6MjmjZtiqSkpMruUoWKj4/H/fffDzc3N/j4+KBr1644ePCgrs3ly5cRFxcHLy8vuLq6onv37sVenOxuMnHiRBgMBgwdOlSbV1XG4fjx4/j3v/8NLy8vODk54d5778WuXdYLfYkI3nnnHfj5+cHJyQkxMTFITU2txB7/c6x71j3Aur+ldV/mj13fYZYuXSpGo1G7lPCAAQPEbDbLyZMnK7trFaZDhw6SkJAgKSkpsmfPHnn00UclMDBQLl68qLUZNGiQ1K5dWzZu3Ci7du2SBx54QJo3b16Jva5YSUlJEhwcLI0aNZIhQ4Zo86vCOGRmZkpQUJD069dPdu7cKYcPH5Z169bJn3/+qbWZOHGieHh4yFdffSV79+6Vzp07S0hIiOTm5lZiz28c6551L8K6v9V1f9cFiCZNmkhcXJz2d0FBgfj7+2sX96kKTp06JQBky5YtIiKSnZ0t1apVkxUrVmhtDhw4IABk+/btldXNCnPhwgUJCwuTDRs2SOvWrbUXkqoyDq+//rq0bNmyxPstFov4+vrKlClTtHnZ2dliMplkyZIlt6KLNx3rnnXPur/1dX9XfYWRn5+P3bt3IyYmRptnZ2eHmJgYbN++vRJ7dmudO3cOQOHPiQPA7t27ceXKFd24REREIDAw8K4cl7i4OHTq1Em3vUDVGYfVq1cjOjoaPXr0gI+PDyIjIzFnzhzt/iNHjiAjI0M3Dh4eHmjatOkdOQ6s+0Kse9b9ra77uypAnDlzBgUFBTYX4alZsyYyMjIqqVe3lsViwdChQ9GiRQs0bNgQAJCRkQGj0Qiz2axrezeOy9KlS/HLL78gPj7e5r6qMg6HDx/GzJkzERYWhnXr1mHw4MF4+eWXMW/ePADQtvVuqRPWPeuedV85dV/hP2VNt1ZcXBxSUlLw448/VnZXbrljx45hyJAh2LBhAxwdHSu7O5XGYrEgOjoaEyZMAABERkYiJSUFs2bNQt++fSu5d1QRWPes+8qo+7tqD4S3tzfs7e1tjq49efIkfH19K6lXt86LL76Ib775Bps2bUKtWrW0+b6+vsjPz0d2drau/d02Lrt378apU6dw3333wcHBAQ4ODtiyZQumTZsGBwcH1KxZs0qMg5+fn811JOrVq4f09HQA0Lb1bqkT1j3rnnVfOXV/VwUIo9GIqKgobNy4UZtnsViwceNGNGvWrBJ7VrFEBC+++CJWrlyJxMREhISE6O6PiopCtWrVdONy8OBBpKen31Xj0q5dO+zbtw979uzRbtHR0ejTp482XRXGoUWLFjan8/3xxx8ICgoCAISEhMDX11c3DufPn8fOnTvvyHFg3bPuWfeVVPc3dOjlbWzp0qViMpnks88+k/3798vAgQPFbDZLRkZGZXetwgwePFg8PDxk8+bNcuLECe2Wk5OjtRk0aJAEBgZKYmKi7Nq1S5o1aybNmjWrxF7fGtcejS1SNcYhKSlJHBwcZPz48ZKamiqLFi0SZ2dnWbhwodZm4sSJYjabZdWqVfLrr79Kly5d7vjTOFn3rPsirPtbU/d3XYAQEZk+fboEBgaK0WiUJk2ayI4dOyq7SxUKQLG3hIQErU1ubq688MIL4unpKc7OzvLEE0/IiRMnKq/Tt8j1LyRVZRy+/vpradiwoZhMJomIiJDZs2fr7rdYLPL2229LzZo1xWQySbt27eTgwYOV1Nubg3XPui/Cur81dc/LeRMREZGyu+oYCCIiIro1GCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlDFAEBERkTIGCCIiIlLGAEFERETKGCCIiIhIGQMEERERKWOAICIiImUMEERERKSMAYKIiIiUMUAQERGRMgYIIiIiUsYAQURERMoYIIiIiEgZAwQREREpY4AgIiIiZQwQREREpIwBgjRjxoyBwWDAmTNnKrsrmn79+iE4OLiyu0F012Ld041igKA7nsViwWeffYbOnTujdu3acHFxQcOGDfHuu+/i8uXLld09Iqogc+bMQevWrVGzZk2YTCaEhIQgNjYWaWlpld21KsGhsjtA9E/l5OQgNjYWDzzwAAYNGgQfHx9s374do0ePxsaNG5GYmAiDwVDZ3SSimyw5ORkhISHo3LkzPD09ceTIEcyZMwfffPMN9u7dC39//8ru4l2NAYLueEajET/99BOaN2+uzRswYACCg4O1EBETE1OJPSSiivC///3PZl7Xrl0RHR2N+fPnY+TIkZXQq6qDX2GQjezsbPTr1w9msxkeHh6IjY1FTk6OTbuFCxciKioKTk5OqF69Op566ikcO3ZM12br1q3o0aMHAgMDYTKZULt2bQwbNgy5ubk26/vqq6/QsGFDODo6omHDhli5cmW5+ms0GnXhocgTTzwBADhw4EC51kNUld1pdV+SomMnsrOz/9F6qGzcA0E2evbsiZCQEMTHx+OXX37B3Llz4ePjg0mTJmltxo8fj7fffhs9e/ZE//79cfr0aUyfPh2tWrVCcnIyzGYzAGDFihXIycnB4MGD4eXlhaSkJEyfPh1//fUXVqxYoa1v/fr16N69O+rXr4/4+HicPXsWsbGxqFWr1g1vR0ZGBgDA29v7htdBVFXcyXV/9uxZFBQUID09HePGjQMAtGvX7p8PCpVOiP6/0aNHCwB59tlndfOfeOIJ8fLy0v5OS0sTe3t7GT9+vK7dvn37xMHBQTc/JyfH5nHi4+PFYDDI0aNHtXmNGzcWPz8/yc7O1uatX79eAEhQUNANbU9MTIy4u7tLVlbWDS1PVBXcDXVvMpkEgAAQLy8vmTZtWrmXpRvHrzDIxqBBg3R/P/jggzh79izOnz8PAPjyyy9hsVjQs2dPnDlzRrv5+voiLCwMmzZt0pZ1cnLSpi9duoQzZ86gefPmEBEkJycDAE6cOIE9e/agb9++8PDw0Nq3b98e9evXv6FtmDBhAr7//ntMnDhR+1RERCW7k+v+u+++w5o1a/D+++8jMDAQly5dUt5+UsevMMhGYGCg7m9PT08AQFZWFtzd3ZGamgoRQVhYWLHLV6tWTZtOT0/HO++8g9WrVyMrK0vX7ty5cwCAo0ePAkCx67vnnnvwyy+/KPV/2bJleOutt/Dcc89h8ODBSssSVVV3ct0/9NBDAICOHTuiS5cuaNiwIVxdXfHiiy+Wex2kjgGCbNjb2xc7X0QAFP7ugsFgwHfffVdsW1dXVwBAQUEB2rdvj8zMTLz++uuIiIiAi4sLjh8/jn79+sFisdz0vm/YsAHPPPMMOnXqhFmzZt309RPdre7kur9WaGgoIiMjsWjRIgaICsYAQcpCQ0MhIggJCUF4eHiJ7fbt24c//vgD8+bNwzPPPKPN37Bhg65dUFAQACA1NdVmHQcPHix3v3bu3IknnngC0dHRWL58ORwc+PQmullu17ovTm5uLvLy8v7ROqhsPAaClHXr1g329vYYO3as9umkiIjg7NmzAKyfaK5tIyKYOnWqbhk/Pz80btwY8+bN03ZvAoUvOPv37y9Xnw4cOIBOnTohODgY33zzje47WCL65263ur969arN1yMAkJSUhH379iE6Orr8G0c3hB/RSFloaCjeffddjBo1CmlpaejatSvc3Nxw5MgRrFy5EgMHDsSIESMQERGB0NBQjBgxAsePH4e7uzu++OKLYos+Pj4enTp1QsuWLfHss88iMzMT06dPR4MGDXDx4sVS+3PhwgV06NABWVlZePXVV/Htt9/a9LdZs2Y3dQyIqprbre4vXryI2rVro1evXmjQoAFcXFywb98+JCQkwMPDA2+//XZFDQUVufUnftDtquh0rtOnT+vmJyQkCAA5cuSIbv4XX3whLVu2FBcXF3FxcZGIiAiJi4uTgwcPam32798vMTEx4urqKt7e3jJgwADZu3evAJCEhASb9dWrV09MJpPUr19fvvzyS+nbt2+Zp3MdOXJEO4WruFvfvn3/wagQ3d3u1LrPy8uTIUOGSKNGjcTd3V2qVasmQUFB8txzz9n0mSqGQeS6fVFEREREZeAxEERERKSMAYKIiIiUMUAQERGRsgoLEJmZmejTpw/c3d1hNpvx3HPPlXlUbZs2bWAwGHS3639elYhuX6x7oqqjwgJEnz598Ntvv2HDhg149tlnMX/+fHh4eKBp06ZISkoqcbkBAwZg9uzZCA0NhclkwtatW7FmzZqK6iYR3URFdf/888/DyckJCQkJCA4OLrXmgcIrJxbVfEREBGJiYm5Rj4noRlVIgDhw4ADWrl2LuXPnIi0tDTNmzMBLL70Ei8WCsLAwdOjQAadOnSp22ezsbAwePBjPP/88kpOT0a1bN3Tt2hUpKSkV0VUiukmK6r5nz56YNm0aJkyYgJkzZ+Ls2bN4+OGHS6z5c+fOITExUav5J598Er1792bNE93mKuQ0zk8//RTDhw9HVlYWmjZtCkdHRxw9ehRHjx5F3bp1kZ2djeHDh2PkyJG65dq0aYMff/wRBQUF+k4aDBg4cGCx1zbIy8vT/WSpxWJBZmYmvLy8YDAYbvamEVUpIoILFy7A398fdnalf94oqvvw8HCt5jMyMpCXlwd3d3eMGjXKpuYBwN3dHRcuXNDNK63mAdY9UUVRqfkK+SGp8ePHS3h4uOTl5YnBYBAHBwf59NNPxdPTU1q0aCFGo1E6dOhgs9zHH38sbm5uYjQa5aOPPhJfX1/p2LGjvPLKK9KoUaNiH6voR1B44423irsdO3asXHUfFhamq/nffvtNHB0dxd7evtiaFxFxdnYWo9EoiYmJWt3XrVu3xJpn3fPGW8XfylPzSnsgRo4ciUmTJpXa5sCBA/jyyy8xb948bNq0CQEBAejevTs+//xz+Pj4YPTo0Rg5ciTMZjOOHTtms7yDgwNMJhMuXbqExMREtGvXDmPGjMH//vc/nDx50qb9+fPntevVF/3doEED1J7+GuycTHilsf4CLrM+6axN+83fp03LPUG6dke6uGnT3vdad736u1ofa//3+svQ1nrQuj15H/lq0xnN9L8Y3rdjojb95Yy22nTmffo9Lz3vt35vvGGu9aeY8930n7C2xM3Uppv80E+b7tVgt67d9jMh2vRfyf7atN01D2t/Sb/uex7+U5s+tiBUm86O0DWDwS9Xm647+qz1cbrW1rWr38V6kZyUNfdo0zmBV3XtXNKsY3bF3foU3f2fT3Tt7v+kvzad523dELt863ZUC9QfxGde6apNZ7S1Pq7BqL9KoO+31ssTu6zaZV23m6uunSHA+r8ucDNp0yeb6Nvl+FvXHzYnQ5v+u4O/rt2FMOt2tIj8XZv+cb/1AkbGU/rnVPD71ufysRcaadNBi4/q2uGaTxTiYr1eyPH23rpm7umFfSi4chm7vxuPshTV/SeffILDhw9rNQ8ANWrUwIULF1CjRo0yax6AVvdeXl44c+ZMsY9XUt2HzhkGe2cT7Le569pfvebSKAFbrM+Hvx7S/4/s/mVdpyHJug77y9Y2ct0FAGokW5/7DuetDXP99Ov+65FrPs1dU3Oh9f7WtTt0wPp8CF1hXbf9mLO6dhkXrP3zmOti7cNl/evIkSeM2nT4rExtOjPK+j9363Nct0zmVwHatM/2bG36i6++0LWLWvicNi3BOdr0lTz9INmftfZh25PW16un+8fq2l32tLY72fWaPUxZJl07Q4G1vuvUt/b90HEfbbrWSv2n52Mdrcs4/m3tn/GcrhnO32N9TYiYZH2+Xq6nr1P7XGu7Y+2s4x/8dbauXdrjZm3a+ZT1tcx7rv64oMOT7rfed81Lt+fGQ9p0ej/9e45vkvX5dmqwdbzcPnfTtTMnWcfo8Hhrfyxp+ueo7/YCXL16Gbu+n4Ds7Gx4eHigNErXwhg+fDj69etXaps6derA19cXp06dQn5+PgAgOjoaV69eRWZmJvz9/REUFITjx4+XuI7Lly8jKChI+yrj999/L7Ht5MmTMX582S9wRHTjkpKS4ObmVuL9RXV/+vRpANAuZFR0wSN/f3+brymuVVTzFosF//rXvwDA5qvMa7HuiW4DZe6juAH79+8XALJ69WoBIPHx8bJu3ToxGAxy/PhxadCggZjN5mKX9fHxkd69e0tycrJMnz5dAIjRaJSIiIhi27/55puVvquHN97u9lt5dmcW1T1QWPMiotV9eHh4uWp+8+bN0rx5cwEgISEhJT4W65433ir2dtO/wlDRsWNH/PXXX0hJSUGbNm1w/PhxREdHY+HChfDw8ICbmxvc3d0xf/58NGnSBIcOHcLixYuxdetWFBQUYMiQIRg2bBj8/f3x888/o0GDBti9e7fN45R0MFW1atUQGBiIY8eOwd3d3Wa5quL8+fOoXbs2x4HjAEB9HETlgCoADz30EDZv3oy2bdti3LhxiI2NRVRUFL755huYzWbs2LED7dq109V9165d4enpifnz5+PXX3/F0KFDkZaWhsjIyGJrHmDdl4XP90Ich0Iq46BS8xV2Oe9FixZh8ODBSElJwebNm9GyZUu88sorGDx4MPLy8nDffffhhx9+wFtvvYWoqCi88MIL+P7775GcnIwLFy5g79696NKlC3x8fPDTTz+hZs2axT6OyWSCyaT/fsxsNmvfj7q7u1fpJ04RjkMhjkMhlXEo63vQay1ZsgR+fn5ITEzEtm3b0LFjRzg5OSEvLw8NGjTAlStXcPDgQUydOhWLFi2C0WiEvb09tm7dirp16yIgIAC1atVCWlpaiTUPsO7Li+NQiONQqLzjUN6ar7AfkqpevTqWLVuGJk2aoFWrVkhPT0eLFi2QnJwMs9mMjh07QkSQn5+PEydOoHbt2tiyZQvOnz+P5cuXw8vLCwsXLsTq1asREBCAiIiIsh+UiCqVr6+vVvM+Pj749ttvkZKSArPZjDZt2iA4OBitW7dGtWqFB6jWrl0be/bswfLlyxEaGoqMjAxkZ2ez5onuABV+LYxXXnkFO3fuxLhx47Bnzx5ERkbiypUriI0tPPo2MDAQfn5+Wvtx48bBw8MD3333HbZv345GjRrhzJkz6N+/f0kPQUS3EdY8UdVQYV9hFOnVqxdOnz6Nd955BxkZGWjcuDHWrl2r7Z5MT0/Xfc+SlZWFAQMGICMjA56enoiKisK2bdtQv359pcc1mUwYPXq0zW7OqobjUIjjUOhWjENl1TzA/3MRjkMhjkOhihqHCjuIkoiIiO5evJw3ERERKWOAICIiImUMEERERKSMAYKIiIiU3ZUBYsaMGQgODoajoyOaNm2KpKSkshe6g8XHx+P++++Hm5sbfHx80LVrVxw8eFDX5vLly4iLi4OXlxdcXV3RvXv3Yi9OdjeZOHEiDAYDhg4dqs2rKuNw/Phx/Pvf/4aXlxecnJxw7733Ytcu6wXBRATvvPMO/Pz84OTkhJiYGKSmplZij/851j3rHmDd39K6L/PHru8wS5cuFaPRqF1KeMCAAWI2m+XkyZOV3bUK06FDB0lISJCUlBTZs2ePPProoxIYGCgXL17U2gwaNEhq164tGzdulF27dskDDzwgzZs3r8ReV6ykpCQJDg6WRo0ayZAhQ7T5VWEcMjMzJSgoSPr16yc7d+6Uw4cPy7p16+TPP//U2kycOFE8PDzkq6++kr1790rnzp0lJCREcnNzK7HnN451z7oXYd3f6rq/6wJEkyZNJC4uTvu7oKBA/P39tYv7VAWnTp0SALJlyxYREcnOzpZq1arJihUrtDYHDhwQALJ9+/bK6maFuXDhgoSFhcmGDRukdevW2gtJVRmH119/XVq2bFni/RaLRXx9fWXKlCnavOzsbDGZTLJkyZJb0cWbjnXPumfd3/q6v6u+wsjPz8fu3bsRExOjzbOzs0NMTAy2b99eiT27tc6dK7zAffXq1QEAu3fvxpUrV3TjEhERgcDAwLtyXOLi4tCpUyfd9gJVZxxWr16N6Oho9OjRAz4+PoiMjMScOXO0+48cOYKMjAzdOHh4eKBp06Z35Diw7gux7ln3t7ru76oAcebMGRQUFNhchKdmzZrIyMiopF7dWhaLBUOHDkWLFi3QsGFDAEBGRgaMRiPMZrOu7d04LkuXLsUvv/yC+Ph4m/uqyjgcPnwYM2fORFhYGNatW4fBgwfj5Zdfxrx58wBA29a7pU5Y96x71n3l1H2F/5Q13VpxcXFISUnBjz/+WNldueWOHTuGIUOGYMOGDXB0dKzs7lQai8WC6OhoTJgwAQAQGRmJlJQUzJo1C3379q3k3lFFYN2z7iuj7u+qPRDe3t6wt7e3Obr25MmT8PX1raRe3TovvvgivvnmG2zatAm1atXS5vv6+iI/Px/Z2dm69nfbuOzevRunTp3CfffdBwcHBzg4OGDLli2YNm0aHBwcULNmzSoxDn5+fjbXkahXrx7S09MBQNvWu6VOWPese9Z95dT9XRUgjEYjoqKisHHjRm2exWLBxo0b0axZs0rsWcUSEbz44otYuXIlEhMTERISors/KioK1apV043LwYMHkZ6efleNS7t27bBv3z7s2bNHu0VHR6NPnz7adFUYhxYtWticzvfHH38gKCgIABASEgJfX1/dOJw/fx47d+68I8eBdc+6Z91XUt3f0KGXt7GlS5eKyWSSzz77TPbv3y8DBw4Us9ksGRkZld21CjN48GDx8PCQzZs3y4kTJ7RbTk6O1mbQoEESGBgoiYmJsmvXLmnWrJk0a9asEnt9a1x7NLZI1RiHpKQkcXBwkPHjx0tqaqosWrRInJ2dZeHChVqbiRMnitlsllWrVsmvv/4qXbp0ueNP42Tds+6LsO5vTd3fdQFCRGT69OkSGBgoRqNRmjRpIjt27KjsLlUoAMXeEhIStDa5ubnywgsviKenpzg7O8sTTzwhJ06cqLxO3yLXv5BUlXH4+uuvpWHDhmIymSQiIkJmz56tu99iscjbb78tNWvWFJPJJO3atZODBw9WUm9vDtY9674I6/7W1D0v501ERETK7qpjIIiIiOjWYIAgIiIiZQwQREREpIwBgoiIiJQxQBAREZEyBggiIiJSxgBBREREyhggiIiISBkDBBERESljgCAiIiJlDBBERESkjAGCiIiIlP0/xKP8vbVtw4oAAAAASUVORK5CYII=\n" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "# -*- coding: utf-8 -*-\n", + "import math, os\n", + "from typing import Optional, Tuple, List\n", + "import numpy as np\n", + "import paddle\n", + "import paddle.nn as nn\n", + "import paddle.nn.functional as F\n", + "from paddle.io import Dataset, DataLoader\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# ============ 基本设置 ============ # 如需 CPU 改为 'cpu'\n", + "os.makedirs(\"viz_out\", exist_ok=True)\n", + "\n", + "# ============ 工具：正弦位置编码 ============\n", + "class SinusoidalPositionalEncoding(nn.Layer):\n", + " def __init__(self, d_model: int, max_len: int = 4096):\n", + " super().__init__()\n", + " pe = np.zeros((max_len, d_model), dtype=\"float32\")\n", + " position = np.arange(0, max_len, dtype=\"float32\")[:, None]\n", + " div_term = np.exp(np.arange(0, d_model, 2, dtype=\"float32\") * (-math.log(10000.0) / d_model))\n", + " pe[:, 0::2] = np.sin(position * div_term)\n", + " pe[:, 1::2] = np.cos(position * div_term)\n", + " self.register_buffer(\"pe\", paddle.to_tensor(pe), persistable=False)\n", + " def forward(self, x): # (B,T,D)\n", + " T = x.shape[1]\n", + " return x + self.pe[:T, :]\n", + "\n", + "# ============ TabM（占位，可替换为你的实现） ============\n", + "class TabMFeatureExtractor(nn.Layer):\n", + " def __init__(self, num_features: int, d_hidden: int = 512, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(num_features, d_hidden), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_hidden, d_hidden), nn.ReLU(),\n", + " )\n", + " self.d_hidden = d_hidden\n", + " def forward(self, x_num: paddle.Tensor):\n", + " return self.net(x_num)\n", + "\n", + "# ============ 3D ResNet18 ============\n", + "class BasicBlock3D(nn.Layer):\n", + " expansion = 1\n", + " def __init__(self, in_planes, planes, stride=(1,1,1), downsample=None):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv3D(in_planes, planes, 3, stride=stride, padding=1, bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(planes)\n", + " self.relu = nn.ReLU()\n", + " self.conv2 = nn.Conv3D(planes, planes, 3, stride=1, padding=1, bias_attr=False)\n", + " self.bn2 = nn.BatchNorm3D(planes)\n", + " self.downsample = downsample\n", + " def forward(self, x):\n", + " identity = x\n", + " out = self.relu(self.bn1(self.conv1(x)))\n", + " out = self.bn2(self.conv2(out))\n", + " if self.downsample is not None:\n", + " identity = self.downsample(x)\n", + " out = self.relu(out + identity)\n", + " return out\n", + "\n", + "class ResNet3D(nn.Layer):\n", + " def __init__(self, block, layers, in_channels=20, base_width=64):\n", + " super().__init__()\n", + " self.in_planes = base_width\n", + " self.conv1 = nn.Conv3D(in_channels, self.in_planes, kernel_size=(3,7,7),\n", + " stride=(1,2,2), padding=(1,3,3), bias_attr=False)\n", + " self.bn1 = nn.BatchNorm3D(self.in_planes)\n", + " self.relu = nn.ReLU()\n", + " self.maxpool = nn.MaxPool3D(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))\n", + " self.layer1 = self._make_layer(block, base_width, layers[0], stride=(1,1,1))\n", + " self.layer2 = self._make_layer(block, base_width*2, layers[1], stride=(2,2,2))\n", + " self.layer3 = self._make_layer(block, base_width*4, layers[2], stride=(2,2,2))\n", + " self.layer4 = self._make_layer(block, base_width*8, layers[3], stride=(2,2,2))\n", + " self.out_dim = base_width*8 # 512\n", + " self.pool = nn.AdaptiveAvgPool3D(output_size=1)\n", + " def _make_layer(self, block, planes, blocks, stride=(1,1,1)):\n", + " downsample = None\n", + " if stride != (1,1,1) or self.in_planes != planes * block.expansion:\n", + " downsample = nn.Sequential(\n", + " nn.Conv3D(self.in_planes, planes * block.expansion, 1, stride=stride, bias_attr=False),\n", + " nn.BatchNorm3D(planes * block.expansion),\n", + " )\n", + " layers = [block(self.in_planes, planes, stride=stride, downsample=downsample)]\n", + " self.in_planes = planes * block.expansion\n", + " for _ in range(1, blocks):\n", + " layers.append(block(self.in_planes, planes))\n", + " return nn.Sequential(*layers)\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.relu(self.bn1(self.conv1(x)))\n", + " x = self.maxpool(x)\n", + " x = self.layer1(x); x = self.layer2(x); x = self.layer3(x); x = self.layer4(x)\n", + " x = self.pool(x) # (B, 512, 1,1,1)\n", + " x = paddle.flatten(x, 1) # (B, 512)\n", + " return x\n", + "\n", + "class Volume3DEncoder(nn.Layer):\n", + " \"\"\"\n", + " 带安全 hook（仅在可求梯度时注册 backward hook），支持 3D Grad-CAM\n", + " \"\"\"\n", + " def __init__(self, in_channels: int = 20, base: int = 64, dropout: float = 0.0):\n", + " super().__init__()\n", + " self.backbone = ResNet3D(BasicBlock3D, [2,2,2,2], in_channels=in_channels, base_width=base)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.out_dim = self.backbone.out_dim # 512\n", + " self._feat = None\n", + " self._grad = None\n", + " def _save_feat_grad(layer, inp, out):\n", + " self._feat = out # (B, 512, D',H',W')\n", + " if getattr(out, \"stop_gradient\", False):\n", + " return\n", + " def _save_grad(grad):\n", + " self._grad = grad\n", + " out.register_hook(_save_grad)\n", + " self.backbone.layer4.register_forward_post_hook(_save_feat_grad)\n", + " def forward(self, x): # (B, C, D, H, W)\n", + " x = self.backbone(x)\n", + " x = self.drop(x)\n", + " return x\n", + "\n", + "# ============ MoE ============\n", + "class ExpertFFN(nn.Layer):\n", + " def __init__(self, d_model, d_ff, dropout=0.1):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " def forward(self, x):\n", + " return self.fc2(self.drop(F.relu(self.fc1(x))))\n", + "\n", + "class MoEConfig:\n", + " def __init__(self, n_experts=8, top_k=1, d_ff=2048, dropout=0.1,\n", + " router_temp=0.5, use_gumbel=False):\n", + " self.n_experts=n_experts; self.top_k=top_k; self.d_ff=d_ff; self.dropout=dropout\n", + " self.router_temp=router_temp; self.use_gumbel=use_gumbel\n", + "\n", + "class MoE(nn.Layer):\n", + " \"\"\"缓存最近一次路由概率/索引，便于可解释与聚类\"\"\"\n", + " def __init__(self, d_model: int, cfg: MoEConfig):\n", + " super().__init__()\n", + " self.cfg = cfg\n", + " self.router = nn.Linear(d_model, cfg.n_experts)\n", + " self.experts = nn.LayerList([ExpertFFN(d_model, cfg.d_ff, cfg.dropout) for _ in range(cfg.n_experts)])\n", + " self.ln = nn.LayerNorm(d_model); self.drop = nn.Dropout(cfg.dropout)\n", + " self.last_router_probs = None\n", + " self.last_topk_idx = None\n", + " def _router_probs(self, logits):\n", + " if self.cfg.use_gumbel and self.training:\n", + " u = paddle.uniform(logits.shape, min=1e-6, max=1-1e-6, dtype=logits.dtype)\n", + " g = -paddle.log(-paddle.log(u)); logits = logits + g\n", + " return F.softmax(logits / self.cfg.router_temp, axis=-1)\n", + " def forward(self, x):\n", + " orig_shape = x.shape\n", + " if len(orig_shape) == 3: B,T,D = orig_shape; X = x.reshape([B*T, D])\n", + " else: X = x\n", + " N,D = X.shape\n", + " logits = self.router(X); probs = self._router_probs(logits)\n", + " topk_val, topk_idx = paddle.topk(probs, k=self.cfg.top_k, axis=-1)\n", + " all_out = paddle.stack([e(X) for e in self.experts], axis=1) # (N,E,D)\n", + " arangeN = paddle.arange(N, dtype='int64')\n", + " picked_list=[]\n", + " for i in range(self.cfg.top_k):\n", + " idx_i = topk_idx[:, i].astype('int64')\n", + " idx_nd = paddle.stack([arangeN, idx_i], axis=1)\n", + " picked_i = paddle.gather_nd(all_out, idx_nd)\n", + " picked_list.append(picked_i)\n", + " picked = paddle.stack(picked_list, axis=1) # (N,k,D)\n", + " w = topk_val / (paddle.sum(topk_val, axis=-1, keepdim=True) + 1e-9)\n", + " Y = paddle.sum(picked * w.unsqueeze(-1), axis=1) # (N,D)\n", + " Y = self.drop(Y); Y = self.ln(Y + X)\n", + " self.last_router_probs = probs.detach()\n", + " self.last_topk_idx = topk_idx.detach()\n", + " if len(orig_shape)==3: Y = Y.reshape([B,T,D])\n", + " return Y\n", + "\n", + "class MoEHead(nn.Layer):\n", + " def __init__(self, d_model=512, cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.moe = MoE(d_model, cfg or MoEConfig())\n", + " self.last_router_probs = None\n", + " self.last_topk_idx = None\n", + " def forward(self, tok):\n", + " y = self.moe(tok.unsqueeze(1)).squeeze(1)\n", + " self.last_router_probs = self.moe.last_router_probs\n", + " self.last_topk_idx = self.moe.last_topk_idx\n", + " return y\n", + "\n", + "# ============ 自实现版 Multi-Head Self-Attention（记录注意力） ============\n", + "class MultiHeadSelfAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_model = d_model\n", + " self.nhead = nhead\n", + " self.d_head = d_model // nhead\n", + " self.Wq = nn.Linear(d_model, d_model)\n", + " self.Wk = nn.Linear(d_model, d_model)\n", + " self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model)\n", + " self.drop = nn.Dropout(dropout)\n", + " self.last_attn = None # (B,H,T,T)\n", + "\n", + " def forward(self, x): # x: (B,T,D)\n", + " B,T,D = x.shape\n", + " q = self.Wq(x); k = self.Wk(x); v = self.Wv(x)\n", + " def split(t): return t.reshape([B, T, self.nhead, self.d_head]).transpose([0,2,1,3]) # (B,H,T,dh)\n", + " qh, kh, vh = split(q), split(k), split(v)\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head) # (B,H,T,T)\n", + " attn = F.softmax(scores, axis=-1)\n", + " self.last_attn = attn.detach()\n", + " ctx = paddle.matmul(attn, vh) # (B,H,T,dh)\n", + " ctx = ctx.transpose([0,2,1,3]).reshape([B, T, D]) # (B,T,D)\n", + " out = self.drop(self.proj(ctx))\n", + " return out # 残差与LN在外面做\n", + "\n", + "# ============ Self-Attention Transformer（用自实现 MHA） ============\n", + "class TransformerEncoderLayerMoE(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, d_ff=1024, dropout=0.1,\n", + " use_moe: bool = True, moe_cfg: MoEConfig = None, capture_attn: bool = True):\n", + " super().__init__()\n", + " self.use_moe = use_moe; self.capture_attn = capture_attn\n", + " self.self_attn = MultiHeadSelfAttention(d_model, nhead, dropout)\n", + " self.ln1 = nn.LayerNorm(d_model); self.do1 = nn.Dropout(dropout)\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model))\n", + " self.do2 = nn.Dropout(dropout)\n", + " self.last_attn = None # (B,H,T,T)\n", + " def forward(self, x): # (B,T,D)\n", + " h = self.ln1(x)\n", + " out = self.self_attn(h) # (B,T,D)\n", + " if self.capture_attn:\n", + " self.last_attn = self.self_attn.last_attn # (B,H,T,T)\n", + " x = x + self.do1(out)\n", + " if self.use_moe:\n", + " x = self.moe(x)\n", + " else:\n", + " x = x + self.do2(self.ffn(x))\n", + " return x\n", + "\n", + "class TemporalTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=4, num_layers=2, d_ff=1024, dropout=0.1,\n", + " max_len=4096, use_moe: bool = True, moe_cfg: MoEConfig = None, capture_attn=True):\n", + " super().__init__()\n", + " self.pos = SinusoidalPositionalEncoding(d_model, max_len=max_len)\n", + " self.layers = nn.LayerList([\n", + " TransformerEncoderLayerMoE(d_model, nhead, d_ff, dropout,\n", + " use_moe=use_moe, moe_cfg=moe_cfg, capture_attn=capture_attn)\n", + " for _ in range(num_layers)\n", + " ])\n", + " self.last_attn_all_layers: List[paddle.Tensor] = []\n", + " def forward(self, x):\n", + " x = self.pos(x)\n", + " self.last_attn_all_layers = []\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " if layer.last_attn is not None:\n", + " self.last_attn_all_layers.append(layer.last_attn) # (B,H,T,T)\n", + " return x\n", + "\n", + "# ============ AFNO(1D) + MoE FFN ============\n", + "class AFNO1DLayer(nn.Layer):\n", + " def __init__(self, d_model: int, modes: int = 32, num_blocks: int = 8, shrink: float = 0.01, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % num_blocks == 0\n", + " self.d_model=d_model; self.modes=modes; self.num_blocks=num_blocks; self.block=d_model//num_blocks\n", + " scale=1.0/math.sqrt(self.block); init = nn.initializer.Uniform(-scale, scale)\n", + " self.w1r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w1i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w2r = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.w2i = self.create_parameter([num_blocks, self.block, self.block], default_initializer=init)\n", + " self.ln = nn.LayerNorm(d_model); self.drop = nn.Dropout(dropout); self.shrink = shrink\n", + " def _cl(self, xr, xi, Wr, Wi):\n", + " out_r = paddle.matmul(xr, Wr) - paddle.matmul(xi, Wi)\n", + " out_i = paddle.matmul(xr, Wi) + paddle.matmul(xi, Wr)\n", + " return out_r, out_i\n", + " def forward(self, x): # (B,T,D)\n", + " B,T,D = x.shape; Kmax=T//2+1; K=min(self.modes, Kmax)\n", + " h=self.ln(x); h_td=h.transpose([0,2,1]); h_ft=paddle.fft.rfft(h_td) # (B,D,F)\n", + " h_ft=h_ft.reshape([B, self.num_blocks, self.block, Kmax])\n", + " xk=h_ft[:,:,:, :K].transpose([0,1,3,2]) # (B,G,K,Cb)\n", + " xr, xi = paddle.real(xk), paddle.imag(xk)\n", + " yr, yi = self._cl(xr, xi, self.w1r, self.w1i)\n", + " yr = F.gelu(yr); yi = F.gelu(yi)\n", + " yr = F.softshrink(yr, threshold=self.shrink); yi = F.softshrink(yi, threshold=self.shrink)\n", + " yr, yi = self._cl(yr, yi, self.w2r, self.w2i)\n", + " yk = paddle.complex(yr, yi).transpose([0,1,3,2]).reshape([B,D,K])\n", + " out_ft = paddle.zeros([B,D,Kmax], dtype='complex64')\n", + " out_ft[:,:, :K] = yk\n", + " out_td = paddle.fft.irfft(out_ft, n=T)\n", + " out = out_td.transpose([0,2,1])\n", + " out = self.drop(out)\n", + " return x + out\n", + "\n", + "class AFNOTransformerFlexible(nn.Layer):\n", + " def __init__(self, d_model=512, num_layers=2, modes=32, dropout=0.1,\n", + " d_ff=1024, use_moe: bool = True, moe_cfg: MoEConfig = None):\n", + " super().__init__()\n", + " self.layers = nn.LayerList([AFNO1DLayer(d_model, modes, 8, 0.01, dropout) for _ in range(num_layers)])\n", + " self.use_moe = use_moe\n", + " if use_moe:\n", + " self.moe = MoE(d_model, moe_cfg or MoEConfig(d_ff=d_ff, dropout=dropout))\n", + " else:\n", + " self.ffn = nn.Sequential(nn.LayerNorm(d_model),\n", + " nn.Linear(d_model, d_ff), nn.ReLU(), nn.Dropout(dropout),\n", + " nn.Linear(d_ff, d_model))\n", + " self.do = nn.Dropout(dropout)\n", + " def forward(self, x):\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " if self.use_moe:\n", + " x = self.moe(x)\n", + " else:\n", + " x = x + self.do(self.ffn(x))\n", + " return x\n", + "\n", + "# ============ Cross-Attention（记录注意力） ============\n", + "class MultiHeadCrossAttention(nn.Layer):\n", + " def __init__(self, d_model: int, nhead: int = 8, dropout: float = 0.1):\n", + " super().__init__()\n", + " assert d_model % nhead == 0\n", + " self.d_head = d_model // nhead; self.nhead = nhead\n", + " self.Wq = nn.Linear(d_model, d_model); self.Wk = nn.Linear(d_model, d_model); self.Wv = nn.Linear(d_model, d_model)\n", + " self.proj = nn.Linear(d_model, d_model); self.drop = nn.Dropout(dropout); self.ln = nn.LayerNorm(d_model)\n", + " self.last_attn = None # (B, H, Nq, Nk)\n", + " def forward(self, q, kv):\n", + " B, Nq, D = q.shape; Nk = kv.shape[1]\n", + " def split(t): return t.reshape([B, -1, self.nhead, self.d_head]).transpose([0,2,1,3])\n", + " qh = split(self.Wq(q)); kh = split(self.Wk(kv)); vh = split(self.Wv(kv))\n", + " scores = paddle.matmul(qh, kh, transpose_y=True) / math.sqrt(self.d_head) # (B,H,Nq,Nk)\n", + " attn = F.softmax(scores, axis=-1)\n", + " self.last_attn = attn.detach()\n", + " ctx = paddle.matmul(attn, vh).transpose([0,2,1,3]).reshape([B,Nq,D])\n", + " out = self.drop(self.proj(ctx))\n", + " return self.ln(out + q)\n", + "\n", + "class BiModalCrossFusion(nn.Layer):\n", + " def __init__(self, d_model=512, nhead=8, dropout=0.1, fuse_hidden=512):\n", + " super().__init__()\n", + " self.ca_v_from_t = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.ca_t_from_v = MultiHeadCrossAttention(d_model, nhead, dropout)\n", + " self.fuse = nn.Sequential(nn.Linear(2*d_model, fuse_hidden), nn.ReLU(), nn.Dropout(dropout))\n", + " self.out_dim = fuse_hidden\n", + " self.last_attn_v_from_t = None # (B,H,1,1)\n", + " self.last_attn_t_from_v = None # (B,H,1,T)\n", + " def forward(self, video_seq, tabm_tok):\n", + " v_tok = video_seq.mean(axis=1, keepdim=True) # (B,1,D)\n", + " t_tok = tabm_tok.unsqueeze(1) # (B,1,D)\n", + " v_upd = self.ca_v_from_t(v_tok, t_tok)\n", + " t_upd = self.ca_t_from_v(t_tok, video_seq)\n", + " self.last_attn_v_from_t = self.ca_v_from_t.last_attn\n", + " self.last_attn_t_from_v = self.ca_t_from_v.last_attn\n", + " fused = paddle.concat([v_upd, t_upd], axis=-1).squeeze(1)\n", + " return self.fuse(fused)\n", + "\n", + "# ============ 总模型 ============\n", + "class TwoModalMultiLabelModel(nn.Layer):\n", + " def __init__(self, vid_channels=20, vid_frames=365, depth_n=24,\n", + " vec_dim=424, d_model=256, nhead=4, n_trans_layers=2, trans_ff=512,\n", + " tabm_hidden=256, dropout=0.1, num_labels=4,\n", + " moe_temporal_attn=True, moe_temporal_afno=True, moe_fused=False, moe_tabm=False,\n", + " afno_modes=32):\n", + " super().__init__()\n", + " self.vol_encoder = Volume3DEncoder(in_channels=vid_channels, dropout=dropout)\n", + " # 关键：3D ResNet 输出 512 → d_model 的输入投影\n", + " self.video_in = nn.Linear(self.vol_encoder.out_dim, d_model)\n", + "\n", + " self.trans_attn = TemporalTransformerFlexible(\n", + " d_model=d_model, nhead=nhead, num_layers=n_trans_layers, d_ff=trans_ff, dropout=dropout,\n", + " max_len=vid_frames, use_moe=moe_temporal_attn, moe_cfg=MoEConfig(d_ff=max(2048,trans_ff), n_experts=8),\n", + " capture_attn=True\n", + " )\n", + " self.trans_afno = AFNOTransformerFlexible(\n", + " d_model=d_model, num_layers=n_trans_layers, modes=afno_modes, dropout=dropout,\n", + " d_ff=trans_ff, use_moe=moe_temporal_afno, moe_cfg=MoEConfig(d_ff=max(2048,trans_ff), n_experts=8)\n", + " )\n", + " self.video_merge = nn.Linear(2*d_model, d_model)\n", + "\n", + " self.tabm = TabMFeatureExtractor(vec_dim, d_hidden=tabm_hidden, dropout=dropout)\n", + " self.tabm_proj = nn.Linear(tabm_hidden, d_model)\n", + " self.moe_tabm = moe_tabm\n", + " if moe_tabm:\n", + " self.tabm_moe = MoEHead(d_model=d_model, cfg=MoEConfig(d_ff=1024, n_experts=6))\n", + "\n", + " self.fusion = BiModalCrossFusion(d_model=d_model, nhead=nhead, dropout=dropout, fuse_hidden=d_model)\n", + " self.moe_fused = moe_fused\n", + " if moe_fused:\n", + " self.fused_moe = MoEHead(d_model=d_model, cfg=MoEConfig(d_ff=1024, n_experts=6))\n", + "\n", + " self.head = nn.Linear(self.fusion.out_dim, num_labels)\n", + " self.depth_n = depth_n\n", + "\n", + " def encode(self, x_video, x_vec):\n", + " B,T,C,H,W,N = x_video.shape\n", + " assert N == self.depth_n\n", + " xvt = x_video.transpose([0,1,2,5,3,4]).reshape([B*T, C, N, H, W])\n", + " f_frame = self.vol_encoder(xvt) # (B*T,512)\n", + " seq = f_frame.reshape([B, T, -1]) # (B,T,512)\n", + " seq = self.video_in(seq) # (B,T,d_model)\n", + "\n", + " z_attn = self.trans_attn(seq) # (B,T,d_model)\n", + " z_afno = self.trans_afno(seq) # (B,T,d_model)\n", + " z_vid = self.video_merge(paddle.concat([z_attn, z_afno], axis=-1)) # (B,T,d_model)\n", + "\n", + " z_tabm = self.tabm(x_vec); z_tabm = self.tabm_proj(z_tabm) # (B,d_model)\n", + " if self.moe_tabm:\n", + " z_tabm = self.tabm_moe(z_tabm)\n", + "\n", + " fused = self.fusion(z_vid, z_tabm) # (B,d_model)\n", + " if self.moe_fused:\n", + " fused = self.fused_moe(fused)\n", + " return fused\n", + "\n", + " def forward(self, x_video, x_vec):\n", + " fused = self.encode(x_video, x_vec)\n", + " logits = self.head(fused)\n", + " return logits\n", + "\n", + "# ============ 3D Grad-CAM ============\n", + "class GradCAM3D:\n", + " def __init__(self, model: TwoModalMultiLabelModel):\n", + " self.model = model\n", + " @paddle.no_grad()\n", + " def _trilinear_upsample(self, vol, out_shape):\n", + " try:\n", + " from scipy.ndimage import zoom\n", + " Dz = out_shape[0] / vol.shape[0]\n", + " Dy = out_shape[1] / vol.shape[1]\n", + " Dx = out_shape[2] / vol.shape[2]\n", + " return zoom(vol, (Dz, Dy, Dx), order=1)\n", + " except Exception:\n", + " return vol\n", + " def generate(self, x_video, x_vec, target_class: int = 0, time_index: int = 0):\n", + " assert x_video.shape[0] == 1, \"Grad-CAM 演示请用单样本 B=1\"\n", + " self.model.eval()\n", + " self.model.clear_gradients()\n", + " logits = self.model(x_video.astype('float32'), x_vec.astype('float32')) # (1,num_labels)\n", + " cls = logits[0, target_class]\n", + " cls.backward()\n", + " feat = self.model.vol_encoder._feat # (1,512,D',H',W')\n", + " grad = self.model.vol_encoder._grad\n", + " assert (feat is not None) and (grad is not None), \"未捕获到特征/梯度\"\n", + " feat_np = feat.numpy()[0]; grad_np = grad.numpy()[0]\n", + " w = grad_np.mean(axis=(1,2,3)) # (512,)\n", + " cam = np.maximum(0, np.tensordot(w, feat_np, axes=(0,0))) # (D',H',W')\n", + " cam = cam - cam.min(); cam = cam / (cam.max() + 1e-8)\n", + " # 将 CAM 插值到输入体素大小：(N,H,W)；这里我们没有逐帧求 CAM，而是对“最后一层体特征”整体做\n", + " # 若你需要对某个 time_index 的体做 CAM，可在 3D 编码处按帧送入并单独反传。\n", + " B,T,C,H,W,N = x_video.shape\n", + " cam_up = self._trilinear_upsample(cam, (N, H, W))\n", + " return cam_up\n", + "\n", + "# ============ MoE 路由聚类工具 ============\n", + "def kmeans_numpy(X: np.ndarray, K: int = 4, iters: int = 50, seed: int = 0):\n", + " rng = np.random.default_rng(seed)\n", + " N,D = X.shape\n", + " cent = X[rng.choice(N, K, replace=False)]\n", + " for _ in range(iters):\n", + " dist2 = ((X[:,None,:]-cent[None,:,:])**2).sum(axis=2) # (N,K)\n", + " idx = dist2.argmin(axis=1)\n", + " new_cent = np.stack([X[idx==k].mean(axis=0) if np.any(idx==k) else cent[k] for k in range(K)], 0)\n", + " if np.allclose(new_cent, cent): break\n", + " cent = new_cent\n", + " return idx, cent\n", + "\n", + "def collect_moe_routing_vectors(model: TwoModalMultiLabelModel, loader: DataLoader,\n", + " branch: str = \"temporal_attn\", topk_hist: bool = True):\n", + " model.eval()\n", + " vecs = []\n", + " for x_vid, x_vec, y in loader:\n", + " _ = model(x_vid.astype('float32'), x_vec.astype('float32'))\n", + " if branch == \"temporal_attn\":\n", + " moe = None\n", + " for lyr in model.trans_attn.layers[::-1]:\n", + " if hasattr(lyr, \"moe\"):\n", + " moe = lyr.moe; break\n", + " elif branch == \"temporal_afno\":\n", + " moe = model.trans_afno.moe if hasattr(model.trans_afno, \"moe\") else None\n", + " elif branch == \"tabm\":\n", + " moe = model.tabm_moe.moe if getattr(model, \"moe_tabm\", False) else None\n", + " else:\n", + " moe = model.fused_moe.moe if getattr(model, \"moe_fused\", False) else None\n", + " if moe is None or moe.last_router_probs is None:\n", + " continue\n", + " probs = moe.last_router_probs.numpy() # (N_tokens, E)\n", + " if topk_hist:\n", + " top1 = moe.last_topk_idx.numpy()[:,0] # (N_tokens,)\n", + " E = probs.shape[1]\n", + " hist = np.bincount(top1, minlength=E).astype(\"float32\")\n", + " hist = hist / (hist.sum() + 1e-9)\n", + " vecs.append(hist)\n", + " else:\n", + " vecs.append(probs.mean(axis=0))\n", + " return np.stack(vecs, 0) if len(vecs)>0 else None\n", + "\n", + "# ============ Toy 数据集 ============\n", + "class ToyTwoModalDataset(Dataset):\n", + " def __init__(self, n: int, seed: int = 0, T: int = 365, C: int = 20, H: int = 20, W: int = 20, N: int = 24):\n", + " super().__init__()\n", + " rng = np.random.default_rng(seed)\n", + " self.video = rng.normal(size=(n, T, C, H, W, N)).astype('float32')\n", + " self.vec = rng.normal(size=(n, 424)).astype('float32')\n", + " vid_hwn = self.video.mean(axis=(3,4,5))\n", + " vid_avg = vid_hwn.mean(axis=1)\n", + " Wv = rng.normal(size=(C,4)); Wt = rng.normal(size=(424,4))\n", + " logits = vid_avg @ Wv + self.vec @ Wt + rng.normal(scale=0.5, size=(n,4))\n", + " probs = 1.0 / (1.0 + np.exp(-logits))\n", + " self.y = (probs > 0.5).astype('float32')\n", + " def __getitem__(self, idx: int):\n", + " return self.video[idx], self.vec[idx], self.y[idx]\n", + " def __len__(self): return len(self.y)\n", + "\n", + "# ============ 小工具：绘图 ============\n", + "def show_heatmap_2d(arr2d: np.ndarray, title: str, save_path: Optional[str] = None):\n", + " plt.figure(); plt.imshow(arr2d, interpolation='nearest'); plt.title(title); plt.colorbar()\n", + " if save_path: plt.savefig(save_path, bbox_inches='tight');\n", + " plt.show(); plt.close()\n", + "\n", + "def show_attention_matrix(attn: np.ndarray, title: str, save_path: Optional[str] = None):\n", + " if attn.ndim == 4 and attn.shape[2] == 1 and attn.shape[3] == 1:\n", + " attn = attn[0,:,0,0][:,None] # (H,1)\n", + " elif attn.ndim == 4 and attn.shape[2] == 1:\n", + " attn = attn[0] # (H,1,T)\n", + " elif attn.ndim == 4:\n", + " attn = attn[0] # (H,T,T)\n", + " plt.figure(figsize=(5,4))\n", + " if attn.ndim == 2:\n", + " plt.imshow(attn, aspect='auto', interpolation='nearest')\n", + " elif attn.ndim == 3:\n", + " H = attn.shape[0]\n", + " cols = int(np.ceil(np.sqrt(H))); rows = int(np.ceil(H/cols))\n", + " fig, axes = plt.subplots(rows, cols, figsize=(3*cols, 3*rows))\n", + " axes = axes.flatten()\n", + " for h in range(H):\n", + " axes[h].imshow(attn[h], interpolation='nearest'); axes[h].set_title(f\"head {h}\")\n", + " for k in range(H, len(axes)): axes[k].axis('off')\n", + " fig.suptitle(title)\n", + " if save_path: fig.savefig(save_path, bbox_inches='tight')\n", + " plt.show(); plt.close(fig); return\n", + " plt.title(title); plt.colorbar()\n", + " if save_path: plt.savefig(save_path, bbox_inches='tight')\n", + " plt.show(); plt.close()\n", + "\n", + "# ============ Demo：可解释可视化 ============\n", + "if __name__ == \"__main__\":\n", + " # 1) 构造“已训练好”的模型（这里随机权重示意）\n", + " model = TwoModalMultiLabelModel(\n", + " vid_channels=20, vid_frames=365, depth_n=24,\n", + " vec_dim=424, d_model=256, nhead=4, n_trans_layers=2, trans_ff=512,\n", + " tabm_hidden=256, dropout=0.1, num_labels=4,\n", + " moe_temporal_attn=True, moe_temporal_afno=True,\n", + " moe_fused=False, moe_tabm=False, afno_modes=32\n", + " )\n", + " model.eval()\n", + "\n", + " # 2) 取一个样本\n", + " toy = ToyTwoModalDataset(n=8, seed=123, T=365, C=20, H=20, W=20, N=24)\n", + " x_video, x_vec, y = toy[0]\n", + " x_video = paddle.to_tensor(x_video[None, ...]) # (1,T,C,H,W,N)\n", + " x_vec = paddle.to_tensor(x_vec[None, ...]) # (1,424)\n", + "\n", + " # 3) 3D Grad-CAM：一次“有梯度”的前向 + 反传（不要 no_grad）\n", + " model.clear_gradients()\n", + " logits = model(x_video.astype('float32'), x_vec.astype('float32'))\n", + " target_class = int(paddle.argmax(logits, axis=-1)[0])\n", + " cam3d = GradCAM3D(model).generate(\n", + " x_video.astype('float32'), x_vec.astype('float32'),\n", + " target_class=target_class, time_index=0\n", + " ) # (N,H,W) or (D',H',W')\n", + "\n", + " # 展示几个深度切片\n", + " Nz = cam3d.shape[0]\n", + " for z in [0, Nz//3, 2*Nz//3, Nz-1]:\n", + " show_heatmap_2d(cam3d[z], f\"Grad-CAM depth={z}\", save_path=f\"viz_out/gradcam_z{z}.png\")\n", + "\n", + " # 4) Self-Attention & Cross-Attention 注意力矩阵\n", + " with paddle.no_grad():\n", + " _ = model.encode(x_video.astype('float32'), x_vec.astype('float32'))\n", + " last_attn_list = model.trans_attn.last_attn_all_layers\n", + " if len(last_attn_list) > 0:\n", + " attn = last_attn_list[-1].numpy() # (B,H,T,T)\n", + " attn_crop = attn[:, :, :64, :64]\n", + " show_attention_matrix(attn_crop, \"Self-Attention (last layer, first 64 tokens)\",\n", + " save_path=\"viz_out/self_attn_lastlayer_64.png\")\n", + " print(\"Self-Attn matrix shape:\", attn.shape)\n", + " else:\n", + " print(\"Self-Attn not captured.\")\n", + " if model.fusion.last_attn_v_from_t is not None:\n", + " show_attention_matrix(model.fusion.last_attn_v_from_t.numpy(),\n", + " \"Cross-Attn v<-t (token→token)\",\n", + " save_path=\"viz_out/cross_attn_v_from_t.png\")\n", + " if model.fusion.last_attn_t_from_v is not None:\n", + " attn_tv = model.fusion.last_attn_t_from_v.numpy()\n", + " attn_tv_crop = attn_tv[:,:,:, :64]\n", + " show_attention_matrix(attn_tv_crop,\n", + " \"Cross-Attn t<-v (token←video_seq first 64)\",\n", + " save_path=\"viz_out/cross_attn_t_from_v_64.png\")\n", + "\n", + " # 5) MoE 路由聚类（示例用 toy 数据）\n", + " def collate_fn(batch):\n", + " vids, vecs, ys = zip(*batch)\n", + " return (paddle.to_tensor(np.stack(vids, 0)),\n", + " paddle.to_tensor(np.stack(vecs, 0)),\n", + " paddle.to_tensor(np.stack(ys, 0)))\n", + " train_loader = DataLoader(toy, batch_size=1, shuffle=False, collate_fn=collate_fn)\n", + " moe_vecs = collect_moe_routing_vectors(model, train_loader, branch=\"temporal_attn\", topk_hist=True)\n", + " if moe_vecs is not None:\n", + " idx, cent = kmeans_numpy(moe_vecs, K=4, iters=100, seed=0)\n", + " print(\"\\n[MoE Routing Clusters @ temporal_attn]\")\n", + " for k in range(4):\n", + " sel = (idx==k)\n", + " if np.any(sel):\n", + " mean_vec = moe_vecs[sel].mean(axis=0)\n", + " dom = int(mean_vec.argmax())\n", + " print(f\" - Cluster {k}: size={int(sel.sum())}, dominant_expert={dom}, mean_dist={np.round(mean_vec,3)}\")\n", + " plt.figure(figsize=(6,4))\n", + " plt.imshow(moe_vecs, aspect='auto', interpolation='nearest')\n", + " plt.title(\"Samples × Experts (routing histogram)\"); plt.xlabel(\"Expert\"); plt.ylabel(\"Sample\")\n", + " plt.colorbar(); plt.savefig(\"viz_out/moe_routing_heatmap.png\", bbox_inches='tight')\n", + " plt.show(); plt.close()\n", + " else:\n", + " print(\"MoE routing not available on selected branch.\")\n" + ], + "metadata": { + "id": "3FGUHDpRVnm6" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git "a/jointContribution/AI_Climate_Diseases/\347\273\223\351\241\271\346\212\245\345\221\212.md" "b/jointContribution/AI_Climate_Diseases/\347\273\223\351\241\271\346\212\245\345\221\212.md" new file mode 100644 index 0000000000..d1a9424161 --- /dev/null +++ "b/jointContribution/AI_Climate_Diseases/\347\273\223\351\241\271\346\212\245\345\221\212.md" @@ -0,0 +1,437 @@ +# 项目信息 + +## 项目名称 +基于多模态深度学习的眼科疾病发病预测：融合 ERA5 气象数据与 UK Biobank 患者特征 + +--- + +## 时间规划（3 个月细化） + +### 第 1 月：数据准备与预处理 +- **第 1 周** + - 熟悉 ERA5-Land 数据接口与变量选择（确认 50 个气象特征变量）。 + - 搭建多进程下载框架，测试不同并发参数下的下载速度与稳定性。 +- **第 2 周** + - 批量下载并缓存 ERA5-Land 数据（覆盖 2010–2020 年英国地区）。 + - 完成数据格式转换（NetCDF → Zarr/HDF5），并进行时空重采样（0.1° → 20×20×24）。 +- **第 3 周** + - 收集 UK Biobank 患者特征（424 维），统计缺失值分布。 + - 尝试多种填补方法（均值/中位数、MICE、KNN、XGBoost），比较效果。 +- **第 4 周** + - 确定最终缺失值填补策略（XGBoost）。 + - 建立 ERA5 与 UKB 患者数据的对齐方案（空间位置 + 时间窗口）。 + - 输出对齐后的多模态训练样本。 + +--- + +### 第 2 月：模型开发与基线训练 +- **第 1 周** + - 搭建 3D ResNet18 编码器，用于逐日气象立体数据的空间特征提取。 + - 实现 TabM 模块，对 UKB 患者表型进行建模。 +- **第 2 周** + - 实现 Transformer 分支（时序注意力）。 + - 实现 AFNO 分支（频域稀疏建模），完成与 Transformer 的特征拼接与融合。 +- **第 3 周** + - 集成 MoE 模块（Transformer FFN、AFNO FFN、TabM projection、融合层可开关）。 + - 完成 Cross-Attention 融合机制，实现双模态交互。 +- **第 4 周** + - 进行基线训练（toy 数据 + 部分真实样本）。 + - 记录多标签分类指标（AUC、F1、Recall、Precision、PR-AUC、Hamming Loss）。 + - 调参与优化：batch size、学习率、专家数量（MoE）。 + +--- + +### 第 3 月：可解释性与总结 +- **第 1 周** + - 实现并测试 3D Grad-CAM，输出气象模态的空间关注区域。 + - 可视化 Self-Attention Heatmap（365 天的时序权重）。 +- **第 2 周** + - 实现 MoE 路由记录与聚类，统计不同专家的样本分布。 + - 探索 domain-specific 专家（如“冬季患者”、“老年患者”）是否自动分离。 +- **第 3 周** + - 整合可解释性结果： + - Grad-CAM（空间热点图） + - Attention Map（时间热点图） + - MoE 聚类（样本-专家分布） + - 撰写可解释性分析小结。 +- **第 4 周** + - 完成整体技术报告撰写（含方法、实验、困难与总结）。 + - 整理成果，准备论文初稿框架。 + +--- + + +## 方案描述 + +### 1. 数据准备 + +我们结合了两类异质模态的数据： + +- **气象模态（ERA5-Land）** + 我们从 ERA5-Land 下载了 **50 个气象与环境变量**（详见附录，例如 *2m 温度、土壤温度层、雪覆盖、降水量、辐射通量、植被指数* 等）。 + - 原始空间分辨率：0.1°（约 10×10 网格覆盖每个城市）。 + - 每个像素包含 **24 个垂直层**（如土壤层、大气层）。 + - 数据被统一 resize 为 **20×20×24** 的体素，每位患者匹配发病前 **365 天**的数据。 + - 我们使用 **多进程并行下载**加快 ERA5-Land 的数据拉取和处理。 + +- **患者模态（UK Biobank）** + 我们从 UK Biobank 中提取了 **424 项患者特征**（包括人口学、生活习惯、临床变量）。 + 对缺失值，我们采用 **XGBoost 预测填补**方法，相比均值/众数填充在临床异质数据上表现更好。 + 每个患者特征通过 **发病日期**和**居住城市**与 ERA5 气象数据对齐，实现时空匹配。 + +任务为 **多标签分类**：预测 **4 种眼科疾病**（如青光眼、白内障、AMD、糖尿病视网膜病变）的发病风险。由于患者可能合并多种眼病，因此使用多标签框架。 + +--- + +### 2. 模型结构 + +整体结构为一个 **双模态神经网络**，用于捕捉 **时空气象模式**和 **个体患者属性**，并在融合时保持可解释性和灵活性。 + +#### 2.1 气象分支：3D ResNet + Transformer + AFNO + +- **3D ResNet18 主干** + 输入为 **365 天 × 50 通道 × 20×20×24 体素**。 + - 采用 **3D 卷积**在 (time × space × depth) 三个维度上同时建模，能够捕捉 **气候随时间的演变、地表到土壤的能量传输、雪深积累**等模式。 + - 输出为 **每日一个 512 维 embedding**。 + +- **时序 Transformer（含 MoE）** + 每日 embedding 输入到 **Transformer 编码器**： + - **自注意力**用于捕捉 365 天长时依赖。 + - **前馈层 (FFN) 替换为 MoE**：每个时间片的 token 被路由到不同专家，使得模型可以专门化处理 **不同气候类型**（如海洋性气候 vs. 大陆性气候）。 + +- **AFNO 分支（Adaptive Fourier Neural Operator）** + 并行使用 **AFNO** 捕捉气象的周期性： + - 将时间序列通过 **FFT** 转换到频率域。 + - 使用 **分块对角的复数线性变换**学习主要频率模式（如季节性波动、短期振荡）。 + - 通过 **soft-shrinkage 稀疏化**抑制非主要频率的噪声。 + - **逆 FFT**还原时域信号。 + 同时，AFNO 分支也加入了 **MoE**，使不同频率特征由不同专家处理，适应气候区域差异。 + +- **气象时序特征融合** + Transformer 与 AFNO 的输出拼接后，经线性层映射回 512 维，得到综合的气象时序表示。 + +--- + +#### 2.2 患者分支：TabM (Tabular Mixture) + +患者 424 维特征通过 **TabM**（Yandex Research 提出）处理。 + +- **核心原理** + TabM 提出了一种 **高效集成 (efficient ensemble)** 机制： + - 主权重矩阵 \(W\) 在所有子模型间共享。 + - 每个子模型（专家）通过一对 **低秩缩放向量** \((r_e, s_e)\) 调节输入/输出： + \[ + y_e = \big[ (x \odot r_e) W^\top \big] \odot s_e + b_e + \] + 其中 \(x\) 为输入特征，\(\odot\) 表示逐元素乘，\(b_e\) 为每个子模型的偏置。 + - 这样可以在几乎不增加参数量的情况下，构建一个内部的“打包集成模型”。 + +- **优点** + - 让模型在面对 **分布差异的亚群体**时更加鲁棒（例如不同生活方式的人群）。 + - 内部集成提高了 **泛化能力**和 **不确定性估计**。 + - 计算成本几乎与单模型相同。 + +我们将 TabM 输出投影到 512 维，与气象分支对齐。 + +--- + +#### 2.3 跨模态融合 + +采用 **双向交叉注意力 (bi-directional cross-attention)**： + +- **气象 → 患者**：患者 token 在气象序列上查询，关注与疾病最相关的时间片。 +- **患者 → 气象**：气象 summary token 在患者 embedding 上查询，将气候解释与个体特征结合。 + +两个更新后的 token 拼接，经 MLP 得到融合表示。 +可选地在这一层加入 **MoE 头**，让模型专门化处理不同疾病亚型。 + +--- + +#### 2.4 分类器 + +最终融合表示经线性分类头，预测 **4 个眼科疾病的风险概率**。 +损失函数使用 **binary cross-entropy with logits**。 + +--- + +### 3. 训练与推理 + +- **损失函数**：带类别权重的 BCE。 + +- **优化器**：Adam，带梯度裁剪。 + +- **推理时的检索增强 (Retrieval-Augmented Inference)** + 推理时，我们利用训练集构建一个特征库： + - 用模型的中间表示作为索引。 + - 在测试样本预测时，检索出 k 个最相似的训练样本（相似度可选 **余弦**或 **欧式距离**）。 + - 计算邻居的平均概率 \(p_{knn}\)。 + - 与模型预测概率 \(p_{model}\) 融合： + \[ + p_{final} = (1-\alpha)\,p_{model} + \alpha\,p_{knn} + \] + +这样能缓解训练/测试分布差异。 + +--- + +### 4. 可解释性 + +我们设计了多层次的可解释机制： + +- **3D Grad-CAM**：可视化气象体素中（纬度 × 经度 × 深度）最关键的区域。 +- **Transformer 注意力图**：显示模型在 365 天中关注的关键时段（如冬季骤降）。 +- **AFNO 频率分析**：指出模型利用的主导频率成分。 +- **MoE 路由可视化**：分析样本被分配到的专家，揭示潜在的病人亚群体或气候模式（例如“雪覆盖驱动的风险群体”）。 + + + +# 项目总结 + +## 已完成工作 +- 搭建多进程 ERA5-Land 数据下载框架，完成 50 个气象变量的收集。 +- 实现数据格式转换与重采样（0.1° → 20×20×24），构建日尺度气象数据立方体。 +- 收集并清洗 UK Biobank 患者特征（424 维），通过 XGBoost 完成缺失值填补。 +- 设计并实现多模态模型框架（3D ResNet + Transformer + AFNO + TabM + MoE + Cross-Attn）。 +- 完成 toy dataset 与真实数据子集的基线训练，验证模型结构可行性。 + +## 遇到的问题及解决方案 +- **数据下载效率低** → 使用多进程并行下载，大幅缩短获取 ERA5 数据的时间。 +- **数据申请受阻**→ 原计划中希望使用UKB基因数据库以及细粒度到经纬度的数据，但是申请流程太长采用城市地理中心，这样原先预设的时空细粒度难以对齐的问题反而不严重了。 +- **缺失值比例高** → 采用 XGBoost 学习型填补方法，相比均值/中位数填补更符合变量间分布关系。 +- **数据对齐复杂** → 按照患者发病时间窗口（365 天）和居住城市坐标匹配 ERA5 数据，构建个体化时空样本。 +- **PaddlePaddle 模块限制** → 例如 `nn.MultiHeadAttention` 不支持 `need_weights` 参数，导致 attention 可解释性实现受限；通过自定义 Cross-Attn 与保存注意力矩阵解决。 +- **气象数据噪声较多** → 采用了传统Transformer和时间序列维度上的AFNO双路transformer提取时域频域特征。 +- **模型需要可解释性方面的贡献** →Transformer 与 MoE 模块默认不输出可解释信息，容易形成“黑箱”；我们通过 自定义 Cross-Attention 权重输出、3D Grad-CAM 空间可视化、时序 Attention Map、AFNO 频域分解 与 MoE 路由可视化 等手段，显式揭示了模型在空间、时间、频率及亚群体层面的关注点。这些改进不仅解决了可解释性瓶颈，也使模型能够为临床专家与政策制定提供透明、可追溯的证据。 +- **计算负担大** → 模型包含 3D CNN + 双 Transformer 分支 + MoE，需依赖 多块A100 GPU 训练；经过分析主要时间复杂度集中在3D CNN上，我们通过气象特征筛选减少3D CNN通道数量，通过取对最后一个维度（小时）平均压缩维度，将3D CNN替换为2D CNN. +## 未来工作计划 + + +在已有的 **多模态深度学习框架** 基础上（融合 ERA5 气象数据与 UK Biobank 等多源患者数据），本研究聚焦于 **显式级联建模**（环境 → 系统 → 暴露 → 生物 → 疾病），构建跨模态、可解释的疾病预测与干预模拟平台。总体目标是： + +1. 在 *Nature Communications* / *Nature Medicine/AAAI/IJCAI* 发表 1–2 篇论文； +2. 提出适用于多模态级联预测的通用框架； +3. 验证模型在 UKB、CKB、FinnGen、BBJ 等多个 Biobank 上的可扩展性； +4. 提供气象与环境干预下的疾病风险模拟； +5. 为城市规划、空气质量控制、疾病防控政策提供量化证据。 + +## 测试样例 +见paddlepaddle文件，考虑到完整数据运行时间过长因此我们再里面提供了仿真数据构成的toydataloader替代。 + + +# Project Information + +## Project Title +Ophthalmic Disease Onset Prediction Based on Multimodal Deep Learning: Integrating ERA5 Meteorological Data and UK Biobank Patient Features + +--- + +## Timeline (Detailed for 3 Months) + +### Month 1: Data Preparation and Preprocessing +- **Week 1** + - Familiarize with ERA5-Land data interfaces and variable selection (confirm 50 meteorological feature variables). + - Build a multi-process downloading framework and test download speed/stability under different concurrency parameters. +- **Week 2** + - Batch download and cache ERA5-Land data (covering UK regions from 2010–2020). + - Complete data format conversion (NetCDF → Zarr/HDF5) and perform spatiotemporal resampling (0.1° → 20×20×24). +- **Week 3** + - Collect UK Biobank patient features (424 dimensions) and analyze missing value distributions. + - Test multiple imputation methods (mean/median, MICE, KNN, XGBoost) and compare performance. +- **Week 4** + - Finalize missing value imputation strategy (XGBoost). + - Establish alignment scheme between ERA5 and UKB patient data (spatial location + time window). + - Output aligned multimodal training samples. + +--- + +### Month 2: Model Development and Baseline Training +- **Week 1** + - Build a 3D ResNet18 encoder for extracting spatial features from daily meteorological volumetric data. + - Implement TabM module for modeling UKB patient phenotypes. +- **Week 2** + - Implement Transformer branch (temporal attention). + - Implement AFNO branch (frequency-domain sparse modeling) and complete feature concatenation and fusion with Transformer outputs. +- **Week 3** + - Integrate MoE modules (Transformer FFN, AFNO FFN, TabM projection, fusion layer with switchable experts). + - Complete Cross-Attention fusion for bimodal interaction. +- **Week 4** + - Conduct baseline training (toy dataset + partial real samples). + - Record multi-label classification metrics (AUC, F1, Recall, Precision, PR-AUC, Hamming Loss). + - Hyperparameter tuning: batch size, learning rate, number of experts (MoE). + +--- + +### Month 3: Explainability and Summary +- **Week 1** + - Implement and test 3D Grad-CAM to visualize spatial attention regions in meteorological modality. + - Visualize Self-Attention Heatmap (temporal weights over 365 days). +- **Week 2** + - Implement MoE routing recording and clustering; analyze sample distributions across experts. + - Explore whether domain-specific experts (e.g., "winter patients," "elderly patients") are automatically separated. +- **Week 3** + - Consolidate explainability results: + - Grad-CAM (spatial hotspots) + - Attention Map (temporal hotspots) + - MoE clustering (sample-expert distribution) + - Draft interpretability analysis summary. +- **Week 4** + - Complete technical report (methods, experiments, challenges, and conclusions). + - Organize results and prepare initial paper framework. + +--- + +## Project Design + +### 1. Data Preparation + +We integrate two heterogeneous modalities: + +- **Meteorological Modality (ERA5-Land)** + Downloaded **50 meteorological and environmental variables** (e.g., *2m temperature, soil temperature layers, snow cover, precipitation, radiation flux, vegetation index*). + - Original spatial resolution: 0.1° (~10×10 grid per city). + - Each pixel includes **24 vertical layers** (e.g., soil, atmosphere). + - Resized to **20×20×24** voxels, with each patient matched to **365 days** of data before disease onset. + - Used **multi-process parallel downloading** to accelerate ERA5-Land retrieval and preprocessing. + +- **Patient Modality (UK Biobank)** + Extracted **424 patient features** (demographics, lifestyle, clinical variables). + For missing values, applied **XGBoost-based predictive imputation**, which outperforms mean/median filling for heterogeneous clinical data. + Patient features are aligned with ERA5 meteorological data via **onset date** and **residential location**, enabling spatiotemporal matching. + +**Task**: Multi-label classification predicting **4 ophthalmic diseases** (e.g., glaucoma, cataract, AMD, diabetic retinopathy). As patients may develop multiple diseases, a multi-label framework is required. + +--- + +### 2. Model Architecture + +The overall design is a **bimodal neural network**, capturing both **spatiotemporal meteorological patterns** and **individual patient attributes**, while ensuring interpretability and flexibility. + +#### 2.1 Meteorological Branch: 3D ResNet + Transformer + AFNO + +- **3D ResNet18 Backbone** + Input: **365 days × 50 channels × 20×20×24 voxels**. + - **3D convolutions** jointly model (time × space × depth), capturing **seasonal changes, soil-atmosphere energy transfer, and snow accumulation**. + - Outputs **one 512-d embedding per day**. + +- **Temporal Transformer (with MoE)** + Daily embeddings are fed into a **Transformer encoder**: + - **Self-attention** captures long-term dependencies across 365 days. + - **FFN replaced by MoE**: each token is routed to different experts, allowing specialized handling of **climate types** (e.g., maritime vs. continental). + +- **AFNO (Adaptive Fourier Neural Operator) Branch** + Models periodicity in parallel: + - Convert sequence via **FFT** to frequency domain. + - Apply **block-diagonal complex linear transforms** to learn dominant frequencies (e.g., seasonal cycles, short-term oscillations). + - Use **soft-shrinkage sparsity** to suppress noise. + - Perform **inverse FFT** to reconstruct. + - Added MoE to allow frequency-specific specialization across regions. + +- **Meteorological Temporal Feature Fusion** + Concatenate Transformer and AFNO outputs, then project back to 512-d, forming integrated meteorological representations. + +--- + +#### 2.2 Patient Branch: TabM (Tabular Mixture) + +Patient 424-d features are modeled with **TabM** (proposed by Yandex Research). + +- **Core Idea** + TabM is an **efficient ensemble mechanism**: + - Weight matrix \(W\) is shared across sub-models. + - Each expert adjusts input/output via low-rank scaling vectors \((r_e, s_e)\): + \[ + y_e = \big[ (x \odot r_e) W^\top \big] \odot s_e + b_e + \] + where \(x\) is input, \(\odot\) is element-wise multiplication, and \(b_e\) is bias. + - Builds an ensemble internally with minimal additional parameters. + +- **Advantages** + - Robust to **population subgroup heterogeneity** (e.g., lifestyle differences). + - Improves **generalization** and **uncertainty estimation**. + - Computationally comparable to a single model. + +TabM outputs are projected to 512-d for alignment with meteorological features. + +--- + +#### 2.3 Cross-Modal Fusion + +Implemented **bi-directional cross-attention**: + +- **Meteorology → Patient**: patient tokens query meteorological sequences to attend to disease-relevant time slices. +- **Patient → Meteorology**: meteorological summary tokens query patient embeddings, linking climate patterns with individual traits. + +Fused tokens are concatenated and passed through MLP. +Optionally, a **MoE head** is added to specialize for disease subtypes. + +--- + +#### 2.4 Classifier + +The fused representation is passed to a linear classifier for **multi-label risk prediction of 4 ophthalmic diseases**. +Loss function: **binary cross-entropy with logits**. + +--- + +### 3. Training and Inference + +- **Loss Function**: BCE with class weights. +- **Optimizer**: Adam with gradient clipping. +- **Retrieval-Augmented Inference (RAI)**: + - Build feature index from training embeddings. + - At inference, retrieve *k* nearest neighbors. + - Compute average probability \(p_{knn}\). + - Combine with model prediction \(p_{model}\): + \[ + p_{final} = (1-\alpha)\,p_{model} + \alpha\,p_{knn} + \] + - Mitigates train-test distribution shift. + +--- + +### 4. Explainability + +Multi-level interpretability mechanisms: + +- **3D Grad-CAM**: highlights key spatial voxels (lat × lon × depth). +- **Transformer Attention Maps**: identify critical time windows (e.g., winter drops). +- **AFNO Frequency Analysis**: reveal dominant periodic components. +- **MoE Routing Visualization**: analyze expert assignments, revealing subgroups (e.g., "snow-driven high-risk patients"). + +--- + +# Project Summary + +## Completed Work +- Built multi-process ERA5-Land data downloading framework; collected 50 meteorological variables. +- Converted and resampled data (0.1° → 20×20×24) to daily meteorological cubes. +- Collected and cleaned UK Biobank patient features (424-d); imputed missing values using XGBoost. +- Designed and implemented multimodal model (3D ResNet + Transformer + AFNO + TabM + MoE + Cross-Attn). +- Conducted baseline training on toy and subset datasets, validating feasibility. + +## Challenges and Solutions +- **Low data download efficiency** → Solved with multi-process parallel downloading. +- **High missing rates** → Addressed via XGBoost predictive imputation, outperforming mean/median. +- **Complex data alignment** → Built spatiotemporal matching pipeline using onset date + residential location. +- **Framework limitation (PaddlePaddle)** → `nn.MultiHeadAttention` lacked `need_weights`; resolved by custom Cross-Attn with weight saving. +- **Need for interpretability contributions** → Tackled the “black box” issue by integrating: + - Cross-Attention weight outputs + - 3D Grad-CAM spatial visualization + - Temporal attention maps + - AFNO frequency decomposition + - MoE routing visualization + These enhancements provided transparency at spatial, temporal, frequency, and subgroup levels, supporting clinical and policy insights. +- **Heavy computational load** → Model combines 3D CNN + dual Transformers + MoE, requiring multi-GPU (A100); solved using gradient clipping, model quantization, and distributed parallel training. + +--- + +## Future Work Plan + +Building on the existing **multimodal deep learning framework** (ERA5 meteorology + UK Biobank features), the research will focus on **explicit cascade modeling** (Environment → System → Exposure → Biology → Disease) to construct an interpretable, multimodal disease prediction and intervention simulation platform. + +**Goals:** +1. Publish 1–2 papers in *Nature Communications*, *Nature Medicine*, AAAI, or IJCAI. +2. Propose a generalizable multimodal cascade prediction framework. +3. Validate scalability across multiple Biobanks (UKB, CKB, FinnGen, BBJ). +4. Provide disease risk simulations under environmental and climate interventions. +5. Deliver quantitative evidence for urban planning, air quality control, and public health policy.