softs-blog.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: PYTORCH_ENABLE_MPS_FALLBACK=1\n"
     ]
    }
   ],
   "source": [
    "%set_env PYTORCH_ENABLE_MPS_FALLBACK=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from datasetsforecast.long_horizon import LongHorizon\n",
    "\n",
    "from neuralforecast.core import NeuralForecast\n",
    "from neuralforecast.losses.pytorch import MAE, MSE\n",
    "from neuralforecast.models import SOFTS, PatchTST, TSMixer, iTransformer\n",
    "\n",
    "from utilsforecast.losses import mae, mse\n",
    "from utilsforecast.evaluation import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(name):\n",
    "    if name == \"ettm1\":\n",
    "        Y_df, *_ = LongHorizon.load(directory='./', group='ETTm1')\n",
    "        Y_df = Y_df[Y_df['unique_id'] == 'OT'] # univariate dataset\n",
    "        Y_df['ds'] = pd.to_datetime(Y_df['ds'])\n",
    "        val_size = 11520\n",
    "        test_size = 11520\n",
    "        freq = '15T'\n",
    "    elif name == \"ettm2\":\n",
    "        Y_df, *_ = LongHorizon.load(directory='./', group='ETTm2')\n",
    "        Y_df['ds'] = pd.to_datetime(Y_df['ds']) \n",
    "        val_size = 11520\n",
    "        test_size = 11520\n",
    "        freq = '15T'\n",
    "\n",
    "    return Y_df, val_size, test_size, freq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Univariate forecasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:torch.distributed.nn.jit.instantiator:Created a temporary directory at /var/folders/g1/xrd363zx571_wq54f3htnbxr0000gn/T/tmpahcfbz0o\n",
      "INFO:torch.distributed.nn.jit.instantiator:Writing /var/folders/g1/xrd363zx571_wq54f3htnbxr0000gn/T/tmpahcfbz0o/_remote_module_non_scriptable.py\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                   | Params\n",
      "---------------------------------------------------------\n",
      "0 | loss          | MAE                    | 0     \n",
      "1 | padder        | ConstantPad1d          | 0     \n",
      "2 | scaler        | TemporalNorm           | 0     \n",
      "3 | enc_embedding | DataEmbedding_inverted | 147 K \n",
      "4 | encoder       | TransEncoder           | 6.8 M \n",
      "5 | projection    | Linear                 | 49.2 K\n",
      "---------------------------------------------------------\n",
      "7.0 M     Trainable params\n",
      "0         Non-trainable params\n",
      "7.0 M     Total params\n",
      "28.105    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dc41cc20ff8474181b92910ec1f66c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61cbd5f8e67e46bca103bc35fbbfdbce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "warning: loc(\"mps_not_equal\"(\"(mpsFileLoc): /AppleInternal/Library/BuildRoots/91a344b1-f985-11ee-b563-fe8bc7981bff/Library/Caches/com.apple.xbs/Sources/MetalPerformanceShadersGraph/mpsgraph/MetalPerformanceShadersGraph/Core/Files/MPSGraphUtilities.mm\":253:0)): 'anec.not_equal_zero' op Invalid configuration for the following reasons: Tensor dimensions N1D1C1H1W34273 are not within supported range, N[1-65536]D[1-16384]C[1-65536]H[1-16384]W[1-16384].\n",
      "warning: loc(\"mps_select\"(\"(mpsFileLoc): /AppleInternal/Library/BuildRoots/91a344b1-f985-11ee-b563-fe8bc7981bff/Library/Caches/com.apple.xbs/Sources/MetalPerformanceShadersGraph/mpsgraph/MetalPerformanceShadersGraph/Core/Files/MPSGraphUtilities.mm\":294:0)): 'anec.not_equal_zero' op Invalid configuration for the following reasons: Tensor dimensions N1D1C1H1W34273 are not within supported range, N[1-65536]D[1-16384]C[1-65536]H[1-16384]W[1-16384].\n",
      "/Users/marcopeix/miniconda3/envs/neuralforecast/lib/python3.10/site-packages/torch/autograd/__init__.py:200: UserWarning: The operator 'aten::sgn.out' is not currently supported on the MPS backend and will fall back to run on the CPU. This may have performance implications. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/mps/MPSFallback.mm:11.)\n",
      "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2674dfc6f53448beba1efe5bb5a9a2a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fea5134bb6dc40ab93f0e892918d8b89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1d1647e47af40cb8561c0f67d93d41c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d48a3e8b4725495eb3bedf530dd54bb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8128bf44a20d43be93da6ad147842498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b343fa603bb41f1a4fbf482c4efde9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab0b66160c244edfbb3411bb91b7ddd5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b41b25ed0d44ab994892e147f0fccd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "131aba4070bc4f65ba02efaa02242193",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5385c0beda324981a5e7f383d310b4b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:`Trainer.fit` stopped: `max_steps=1000` reached.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da8d36d54d6f457f9c56d49fc519757b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                     | Params\n",
      "-----------------------------------------------------------\n",
      "0 | loss          | MAE                      | 0     \n",
      "1 | padder        | ConstantPad1d            | 0     \n",
      "2 | scaler        | TemporalNorm             | 0     \n",
      "3 | norm          | ReversibleInstanceNorm1d | 2     \n",
      "4 | mixing_layers | Sequential               | 169 K \n",
      "5 | out           | Linear                   | 27.7 K\n",
      "-----------------------------------------------------------\n",
      "196 K     Trainable params\n",
      "0         Non-trainable params\n",
      "196 K     Total params\n",
      "0.788     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "095d2962219f4c6a84e427d18a15dff6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "682e20aac872446aacac4597352e881f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be472a3df2b34f53983e1364aaf7a46e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a3ca9f292be4fc5880a6d8bfcc77629",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2c730addf37449586f32662420714cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a3d4389580b4322b6cdbbe7f133650b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb513ce63c9748589b38faee7ba549b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "349a377059aa4ff2ad8cc8fce922f965",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72d880ed95524fbfa11e131997134bf9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9763979119684bcab7edd4374ff0bc54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b05498956f4042ef855c79a7d109ab5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9dbff3b7a864bceab7b39ea37d3e966",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:`Trainer.fit` stopped: `max_steps=1000` reached.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66b73e356b0943d4a3fcaf9a596b5e0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                   | Params\n",
      "---------------------------------------------------------\n",
      "0 | loss          | MAE                    | 0     \n",
      "1 | padder        | ConstantPad1d          | 0     \n",
      "2 | scaler        | TemporalNorm           | 0     \n",
      "3 | enc_embedding | DataEmbedding_inverted | 147 K \n",
      "4 | encoder       | TransEncoder           | 6.3 M \n",
      "5 | projector     | Linear                 | 49.2 K\n",
      "---------------------------------------------------------\n",
      "6.5 M     Trainable params\n",
      "0         Non-trainable params\n",
      "6.5 M     Total params\n",
      "26.012    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d37caef6bc8466c96da03bb3fea1d7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ed6d34599e94aa0af44af20d7e4062b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "140f9cabb61c42a2acdd6f9bea3871ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9baa26672e0d49e7a309abc8e2e99286",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c07a9d5fb84a442584cb7e869864e692",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d71d968a3ab4bfdaaaa9eaa285d4ba0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ac1206993b743069345ed5bcd796744",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2af32968fddf472483d3ed3802fadb40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c4705139042427bbc4c43e04294bdca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcf26cd357a94e4e9678027b6a4bdb30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "724f30ec4e3441628aa3f4d9cad2570a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e48fe89dde442a5ac2151b3ad6893df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:`Trainer.fit` stopped: `max_steps=1000` reached.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e65211ffa89407c8b77cb169a142cb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name         | Type              | Params\n",
      "---------------------------------------------------\n",
      "0 | loss         | MAE               | 0     \n",
      "1 | padder_train | ConstantPad1d     | 0     \n",
      "2 | scaler       | TemporalNorm      | 0     \n",
      "3 | model        | PatchTST_backbone | 846 K \n",
      "---------------------------------------------------\n",
      "846 K     Trainable params\n",
      "3         Non-trainable params\n",
      "846 K     Total params\n",
      "3.387     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "981ccb16c9ce4ea8badd8f4075cb8261",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/marcopeix/dev/neuralforecast/neuralforecast/common/_base_windows.py:325: UserWarning: MPS: no support for int64 repeats mask, casting it to int32 (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/native/mps/operations/Repeat.mm:236.)\n",
      "  y_scale = torch.repeat_interleave(y_scale, repeats=y_hat.shape[-1], dim=-1).to(\n",
      "/Users/marcopeix/dev/neuralforecast/neuralforecast/common/_base_windows.py:558: UserWarning: MPS: no support for int64 reduction ops, casting it to int32 (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/native/mps/operations/ReduceOps.mm:144.)\n",
      "  batch_size = torch.sum(batch_sizes)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34d048ec1793476eab6927e777634792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76db83b2d56a404da781266af3772a65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e185cba28735411d99346f2702ec93d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43f405894ec44493940fef78b2568287",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eec7556f55824f929283a95d6cfc4d1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fc66bdc27c64b5ca2399b52673b3dcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "240b00ddd6e34f0c842602483180f8af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f76bfbbb21b943108238b533d1980bd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22d8b8afad5344f3839c8fd103eff5bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/marcopeix/dev/neuralforecast/neuralforecast/core.py:199: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "Y_df, val_size, test_size, freq = load_data('ettm1')\n",
    "\n",
    "horizon = 96\n",
    "\n",
    "models = [\n",
    "    SOFTS(h=horizon, input_size=3*horizon, n_series=1, max_steps=1000, early_stop_patience_steps=3),\n",
    "    TSMixer(h=horizon, input_size=3*horizon, n_series=1, max_steps=1000, early_stop_patience_steps=3),\n",
    "    iTransformer(h=horizon, input_size=3*horizon, n_series=1, max_steps=1000, early_stop_patience_steps=3),\n",
    "    PatchTST(h=horizon, input_size=3*horizon, max_steps=1000, early_stop_patience_steps=3)\n",
    "]\n",
    "\n",
    "nf = NeuralForecast(models=models, freq=freq)\n",
    "nf_preds = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, n_windows=None)\n",
    "nf_preds = nf_preds.reset_index()\n",
    "\n",
    "ettm1_evaluation = evaluate(df=nf_preds, metrics=[mae, mse], models=['SOFTS', 'TSMixer', 'iTransformer', 'PatchTST'])\n",
    "ettm1_evaluation.to_csv('ettm1_results.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAJOCAYAAACqS2TfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ9UlEQVR4nO3df1xW5eH/8TcivxRFFEV0gKhNMbUUPjUosj4phq2sNJ06rBSLsCGyViL+mqY2NCVTUUlEK80W2lpjKTpxOlkqyrJkzZqK6c0UK9RIQDjfP/xwf70HmCiHW/D1fDzO48F93de5znXddfB+c51zHQfDMAwBAAAAAIB618zeHQAAAAAAoKkidAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYJLm9u7AzaiyslKnTp1Sq1at5ODgYO/uAAAAAABuMoZh6Pz58+rUqZOaNat9PpvQXYNTp07J19fX3t0AAAAAANzkTpw4oZ/85Ce1vk/orkGrVq0kXf7wWrdubefeAAAAAABuNufOnZOvr681P9aG0F2DqkvKW7duTegGAAAAANTqx25JZiE1AAAAAABMQugGAAAAAMAkhG4AAAAAAEzCPd0AAAAA0MRVVFSovLzc3t1oVJycnOTo6HjD7RC6AQAAAKCJMgxDhYWF+u677+zdlUapTZs26tix448ulnY1hG4AAAAAaKKqAneHDh3UokWLGwqPtxLDMFRSUqLTp09Lknx8fK67LUI3AAAAADRBFRUV1sDdrl07e3en0XFzc5MknT59Wh06dLjuS81ZSA0AAAAAmqCqe7hbtGhh5540XlWf3Y3cD0/oBgAAMMHy5csVEBAgV1dXBQUFadeuXbXWtVgsGj16tHr06KFmzZopLi6uWp3U1FSFhYXJ09NTnp6eGjhwoPbu3WtT5/z584qLi5O/v7/c3NwUGhqqffv21ffQADQyXFJ+/erjsyN0AwAA1LONGzcqLi5OiYmJOnjwoMLCwhQREaGCgoIa65eWlqp9+/ZKTEzUHXfcUWOd7OxsjRo1Sjt27FBOTo78/PwUHh6ukydPWutERUUpKytLb731lg4dOqTw8HANHDjQpg4AoGE5GIZh2LsTN5tz587Jw8NDxcXFat26tb27AwAAGpm7775b/fv3V0pKirUsMDBQjz32mObPn3/Vfe+//37deeedSk5Ovmq9iooKeXp6aunSpRo7dqx++OEHtWrVSn/4wx/08MMPW+vdeeed+vnPf65XXnnlhsYEoPG5ePGijh49ar3q5krFxcUqKSlpsL60aNFCHh4eDXa8+nK1z/BacyMLqQEAANSjsrIy5ebmasqUKTbl4eHh2rNnT70dp6SkROXl5Wrbtq0k6dKlS6qoqKj2pdDNzU27d++ut+MCaPyKi4u1ZMkyVVY23HO7mzVzUmzsxEYZvG8UoRsAAKAeFRUVqaKiQt7e3jbl3t7eKiwsrLfjTJkyRZ07d9bAgQMlSa1atVJISIjmzJmjwMBAeXt7a8OGDfrkk09022231dtxATR+JSUlqqwsV0bG4yoqam/68by8zmjYsM0qKSkhdAMAAKB+/PfiO4Zh1NtiRklJSdqwYYOys7NtZrbfeustjRs3Tp07d5ajo6P69++v0aNH68CBA/VyXABNS1FRe1ks1//8aTPdf//96tOnjxwdHbV27Vo5Oztrzpw5GjNmjF544QW9//776tChg5YuXaqIiAhVVFTo2Wef1V/+8hcVFhbKz89PMTExmjRpkk27a9asUVJSko4ePaouXbooNjZWMTExpo6FhdQAAADqkZeXlxwdHavNap8+fbra7Pf1WLhwoebNm6etW7eqb9++Nu9169ZNO3fu1IULF3TixAnt3btX5eXlCggIuOHjAkBDW7t2rby8vLR371796le/0vPPP68nn3xSoaGhOnDggAYPHqzIyMj/m7mv1E9+8hO99957Onz4sGbMmKGpU6fqvffes7aXmpqqxMREzZ07V/n5+Zo3b56mT5+utWvXmjoOQjcAAEA9cnZ2VlBQkLKysmzKs7KyFBoaekNtL1iwQHPmzNHHH3+s4ODgWuu1bNlSPj4++vbbb7VlyxYNHTr0ho4LAPZwxx13aNq0abrtttuUkJAgNzc3eXl5acKECbrttts0Y8YMnT17Vp9++qmcnJz029/+Vv/zP/+jgIAAjRkzRk8//bRN6J4zZ45ee+01PfHEEwoICNATTzyhyZMna+XKlaaOg8vLAQAA6ll8fLwiIyMVHByskJAQrVq1SgUFBYqOjpYkJSQk6OTJk1q3bp11n7y8PEnShQsXdObMGeXl5cnZ2Vm9evWSdPmS8unTp2v9+vXq0qWLdSbd3d1d7u7ukqQtW7bIMAz16NFDX375pX7zm9+oR48eeuaZZxpw9ABQP668msfR0VHt2rVTnz59rGVVVw+dPn1akrRixQq9+eabOn78uH744QeVlZXpzjvvlCSdOXNGJ06c0Pjx4zVhwgRrG5cuXTL9PnNCNwAAQD0bOXKkzp49q9mzZ8tisah3797KzMyUv7+/JMlisVR7Zne/fv2sP+fm5mr9+vXy9/fXsWPHJEnLly9XWVmZhg8fbrPfzJkzNWvWLEmXVyROSEjQ119/rbZt22rYsGGaO3eunJyczBssAJjkv393OTg42JRVrZNRWVmp9957T5MnT9Zrr72mkJAQtWrVSgsWLNAnn3xirSNdvsT87rvvtmnX0dHRzGEQugEAAMwQExNT6+I86enp1coMw7hqe1Xh+2pGjBihESNGXEv3AKBJ2bVrl0JDQ21+73711VfWn729vdW5c2f9+9//1pgxYxq0b4RuAAAAAECj1r17d61bt05btmxRQECA3nrrLe3bt89mIclZs2YpNjZWrVu3VkREhEpLS7V//359++23io+PN61vhG4AAAAAuAV5eZ1pMseJjo5WXl6eRo4cKQcHB40aNUoxMTH685//bK0TFRWlFi1aaMGCBXrppZfUsmVL9enTR3Fxcab2zcH4sWuZbkHnzp2Th4eHiouL1bp1a3t3BwAANBLFxcUqKSmxdzduSIsWLUxfVAhAw7h48aKOHj2qgIAAubq6WsuLi4u1ZMkyVVaWN1hfmjVzUmzsxEb3+6W2z1C69tzITDcAAEA9sMeXWDM01i/GAK6dh4eHYmMnNugfCW/lP+gRugEAAOpBSUmJKivLlZHxuIqK2tu7O9fFy+uMhg3brJKSklv2yzFwq/Dw8OA8byCEbgAAgHpUVNReFouPvbsBALhJNLN3BwAAAAAAaKoI3QAAAAAAmITQ3YQsX77cuqpeUFCQdu3aVWtdi8Wi0aNHq0ePHmrWrFmNy+SnpqYqLCxMnp6e8vT01MCBA7V3795a25w/f74cHBxMX3IfAAAAABoLQncTsXHjRsXFxSkxMVEHDx5UWFiYIiIiVFBQUGP90tJStW/fXomJibrjjjtqrJOdna1Ro0Zpx44dysnJkZ+fn8LDw3Xy5Mlqdfft26dVq1apb9++9TouAAAAAGjMCN1NxKJFizR+/HhFRUUpMDBQycnJ8vX1VUpKSo31u3Tpotdff11jx46tddXCd955RzExMbrzzjvVs2dPpaamqrKyUtu3b7epd+HCBY0ZM0apqany9PSs97EBAAAAQGNF6G4CysrKlJubq/DwcJvy8PBw7dmzp96OU1JSovLycrVt29amfOLEiXr44Yc1cODAejsWAAAAADQFPDKsCSgqKlJFRYW8vb1tyr29vVVYWFhvx5kyZYo6d+5sE67fffddHThwQPv27au34wC3uuXLl2vBggWyWCy6/fbblZycrLCwsBrrWiwW/frXv1Zubq6OHDmi2NhYJScn29RJTU3VunXr9Nlnn0mSgoKCNG/ePN11113WOikpKUpJSdGxY8ckSbfffrtmzJihiIgIU8YIAADsq7i4WCUlJQ12vBYtWtyyzwUndDchDg4ONq8Nw6hWdr2SkpK0YcMGZWdny9XVVZJ04sQJTZo0SVu3brWWAbgxVeszLF++XPfcc49WrlypiIgIHT58WH5+ftXqX7k+w+LFi2tss2p9htDQULm6uiopKUnh4eH6/PPP1blzZ0nST37yE7366qvq3r27JGnt2rUaOnSoDh48qNtvv928AQMAgAZXXFysJUuXqPJSZYMds1nzZop9IfaWDN6E7ibAy8tLjo6O1Wa1T58+XW32+3osXLhQ8+bN07Zt22wWSsvNzdXp06cVFBRkLauoqNBf//pXLV26VKWlpXJ0dLzh4wO3kivXZ5Ck5ORkbdmyRSkpKZo/f361+lXrM0hSWlpajW2+8847Nq9TU1P1/vvva/v27Ro7dqwk6ZFHHrGpM3fuXKWkpOjvf/87oRsAgCampKRElZcqlaEMFanI9ON5yUvDLg1TSUnJNYfu+++/X3369JGjo6PWrl0rZ2dnzZkzR2PGjNELL7yg999/Xx06dNDSpUsVERGhb7/9Vi+88IK2bt2qCxcu6Cc/+YmmTp2qZ555RpJ08uRJxcfHa+vWrWrWrJnuvfdevf766+rSpYuJI7+M0N0EODs7KygoSFlZWXr88cet5VlZWRo6dOgNtb1gwQK98sor2rJli4KDg23ee/DBB3Xo0CGbsmeeeUY9e/bUyy+/TOAG6qhqfYYpU6bYlDfU+gxVKioq9Pvf/17ff/+9QkJC6u24AADg5lKkIllksXc3arV27Vq99NJL2rt3rzZu3Kjnn39eH3zwgR5//HFNnTpVixcvVmRkpAoKCjR9+nQdPnxYf/7zn+Xl5aUvv/xSP/zwg6TL330eeOABhYWF6a9//auaN2+uV155RQ899JA+/fRTOTs7mzoOQncTER8fr8jISAUHByskJESrVq1SQUGBoqOjJUkJCQk6efKk1q1bZ90nLy9P0uXVx8+cOaO8vDw5OzurV69eki5fUj59+nStX79eXbp0sc6ku7u7y93dXa1atVLv3r1t+tGyZUu1a9euWjmAH2fP9Rkk6dChQwoJCdHFixfl7u6uzZs3W38fAAAANLQ77rhD06ZNk3Q5z7z66qvy8vLShAkTJEkzZsxQSkqKPv30UxUUFKhfv37WicIrZ7DfffddNWvWTG+++ab19ts1a9aoTZs2ys7OrrYgdX0jdDcRI0eO1NmzZzV79mxZLBb17t1bmZmZ8vf3l3R5saX/fmZ3v379rD/n5uZq/fr18vf3ty6ktHz5cpWVlWn48OE2+82cOVOzZs0ydTzArayh12eo0qNHD+Xl5em7775TRkaGnnrqKe3cuZPgDQAA7OLKW1sdHR3Vrl079enTx1pWNVFx+vRpPf/88xo2bJgOHDig8PBwPfbYYwoNDZV0Oet8+eWXatWqlU37Fy9e1FdffWX6OAjdTUhMTIxiYmJqfC89Pb1amWEYV22vKnzXRXZ2dp33AXCZvdZnqOLs7GxdSC04OFj79u3T66+/rpUrV97wsQEAAOrKycnJ5rWDg4NNWdWkRGVlpSIiInT8+HH96U9/0rZt2/Tggw9q4sSJWrhwoSorKxUUFFRtnRtJat++vbmDEM/pBoCbxpXrM1wpKyvL+pfa67VgwQLNmTNHH3/8cbX1GWpjGIZKS0tv6LgAAAANpX379nr66af19ttvKzk5WatWrZIk9e/fX0eOHFGHDh3UvXt3m60hVlMndAPATSQ+Pl5vvvmm0tLSlJ+fr8mTJ1dbn6FqxfEqeXl5ysvLs1mf4fDhw9b3k5KSNG3aNKWlpVnXZygsLNSFCxesdaZOnapdu3bp2LFjOnTokBITE5Wdna0xY8Y0zMABALhJLV++XAEBAXJ1dVVQUJB27dpVa12LxaLRo0erR48eatasmeLi4qrVSU1NVVhYmDw9PeXp6amBAwdq7969NnX++te/6pFHHlGnTp3k4OCgDz74oJ5H1fTMmDFDf/jDH/Tll1/q888/10cffaTAwEBJ0pgxY+Tl5aWhQ4dq165dOnr0qHbu3KlJkybp66+/Nr1vXF7eyDX0Q+3rW4sWLW7JZ/UBtbHX+gz/+c9/FBkZKYvFIg8PD/Xt21cff/yxBg0aZN5gAQC4yW3cuFFxcXFavny57rnnHq1cuVIRERE6fPiw/Pz8qtUvLS1V+/btlZiYqMWLF9fYZnZ2tkaNGqXQ0FC5uroqKSlJ4eHh+vzzz9W5c2dJ0vfff6877rhDzzzzjIYNG2ba+LzkZVrbDX0cZ2dnJSQk6NixY3Jzc1NYWJjeffddSZczx1//+le9/PLLeuKJJ3T+/Hl17txZDz74oFq3bm163xyMH7ux9xZ07tw5eXh4qLi4uEH+I1yv4uJiLVmyTJWV5fbuynVr1sxJsbETCd4AgEbPYrFo1apVWrnyWVksPvbuznXx8bHouedW6dlnn5WPT+McA1Cf7r77bvXv318pKSnWssDAQD322GOaP3/+Vfe9//77deeddyo5Ofmq9SoqKuTp6amlS5dWu5pNunzf8ubNm/XYY4/Vuf8XL17U0aNHrTP1VYqLi7Vk6RJVXqqsc5vXq1nzZop9IbbRfe+v7TOUrj03MtPdiJWUlKiyslwZGY+rqMj8BQDqm5fXGQ0btlklJSWN7uQDAABA01ZWVqbc3FxNmTLFpjw8PFx79uypt+OUlJSovLxcbdu2rbc2f4yHh4diX4ht0Ctmb+UrXAndTUBRUftG+xd1AAAA4GZUVFSkioqKak8Q8fb2rvakkRsxZcoUde7cWQMHDqy3Nq+Fh4fHLRuCGxqhGwBuAo19fQbp1v4LNgCg6ap6LFUVwzCqlV2vpKQkbdiwQdnZ2dUuXUbTQegGADtrCuszSKzRAABoWry8vOTo6FhtVvv06dPVZr+vx8KFCzVv3jxt27ZNffv2veH2cPMidAOAnTX29Rkk1mgAADQ9zs7OCgoKUlZWlh5//HFreVZWloYOHXpDbS9YsECvvPKKtmzZouDg4BvtKm5yhG4AuEmwPgMAADeX+Ph4RUZGKjg4WCEhIVq1apUKCgoUHR0tSUpISNDJkye1bt066z55eXmSpAsXLujMmTPKy8uTs7OzevXqJenyJeXTp0/X+vXr1aVLF+tMuru7u9zd3a37fvnll9Y2jx49qry8PLVt27bGR5X9GB5Ydf3q47MjdAMAAABADUaOHKmzZ89q9uzZslgs6t27tzIzM+Xv7y/p8qMCCwoKbPbp16+f9efc3FytX79e/v7+OnbsmCRp+fLlKisr0/Dhw232mzlzpmbNmiVJ2r9/vx544AHre/Hx8ZKkp556Sunp6dfcfycnJ0mXr6pzc3O75v3w/1WtuVP1WV4PQjcAAAAA1CImJkYxMTE1vldTAP6xmdGq8H01999/f73MsDo6OqpNmzY6ffq0pMuLntbXInBNnWEYKikp0enTp9WmTRs5Ojped1uEbgAAAABoojp27ChJ1uCNumnTpo31M7xehG4AAAAAaKIcHBzk4+OjDh06qLy8cT8ppaE5OTnd0Ax3FUI3AAAAAFyhuLjYei9vY9WiRQubJ4o4OjrWS4BE3dk9dC9fvlwLFiyQxWLR7bffruTkZIWFhdVY12Kx6Ne//rVyc3N15MgRxcbGKjk52aZOamqq1q1bp88++0ySFBQUpHnz5umuu+4yeygAAAAAGrni4mItWbJMlZWNe1a4WTMnxcZO5FGeNwG7hu6NGzcqLi5Oy5cv1z333KOVK1cqIiJChw8frnEp/NLSUrVv316JiYlavHhxjW1mZ2dr1KhRCg0Nlaurq5KSkhQeHq7PP/9cnTt3NntIAAAAABqxkpISVVaWKyPjcRUVtbd3d66Ll9cZDRu2WSUlJYTum4BdQ/eiRYs0fvx4RUVFSZKSk5O1ZcsWpaSkaP78+dXqd+nSRa+//rokKS0trcY233nnHZvXqampev/997V9+3aNHTu2nkcAAAAAoCkqKmovi8XH3t1AE9DMXgcuKytTbm6uwsPDbcrDw8O1Z8+eejtOSUmJysvL1bZt21rrlJaW6ty5czYbAAAAAAA3ym6hu6ioSBUVFfL29rYp9/b2VmFhYb0dZ8qUKercubMGDhxYa5358+fLw8PDuvn6+tbb8QEAAAAAty67he4q//1wdsMw6u2B7UlJSdqwYYM2bdokV1fXWuslJCSouLjYup04caJejg8AAAAAuLXZ7Z5uLy8vOTo6VpvVPn36dLXZ7+uxcOFCzZs3T9u2bVPfvn2vWtfFxUUuLi43fEwAAAAAAK5kt5luZ2dnBQUFKSsry6Y8KytLoaGhN9T2ggULNGfOHH388ccKDg6+obYAAAAAALhedl29PD4+XpGRkQoODlZISIhWrVqlgoICRUdHS7p82ffJkye1bt066z55eXmSpAsXLujMmTPKy8uTs7OzevXqJenyJeXTp0/X+vXr1aVLF+tMuru7u9zd3Rt2gAAAAACAW5pdQ/fIkSN19uxZzZ49WxaLRb1791ZmZqb8/f0lSRaLRQUFBTb79OvXz/pzbm6u1q9fL39/fx07dkyStHz5cpWVlWn48OE2+82cOVOzZs0ydTwAAAAAAFzJrqFbkmJiYhQTE1Pje+np6dXKDMO4antV4RsAAAAAAHuz++rlAAAAAAA0VYRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJHYP3cuXL1dAQIBcXV0VFBSkXbt21VrXYrFo9OjR6tGjh5o1a6a4uLga62VkZKhXr15ycXFRr169tHnzZpN6DwAAAABA7ewaujdu3Ki4uDglJibq4MGDCgsLU0REhAoKCmqsX1paqvbt2ysxMVF33HFHjXVycnI0cuRIRUZG6h//+IciIyM1YsQIffLJJ2YOBQAAAACAauwauhctWqTx48crKipKgYGBSk5Olq+vr1JSUmqs36VLF73++usaO3asPDw8aqyTnJysQYMGKSEhQT179lRCQoIefPBBJScnmzgSAAAAAACqs1voLisrU25ursLDw23Kw8PDtWfPnutuNycnp1qbgwcPvmqbpaWlOnfunM0GAAAAAMCNslvoLioqUkVFhby9vW3Kvb29VVhYeN3tFhYW1rnN+fPny8PDw7r5+vpe9/EBAAAAAKhi94XUHBwcbF4bhlGtzOw2ExISVFxcbN1OnDhxQ8cHAAAAAECSmtvrwF5eXnJ0dKw2A3369OlqM9V10bFjxzq36eLiIhcXl+s+JgAAAAAANbHbTLezs7OCgoKUlZVlU56VlaXQ0NDrbjckJKRam1u3br2hNgEAAAAAuB52m+mWpPj4eEVGRio4OFghISFatWqVCgoKFB0dLenyZd8nT57UunXrrPvk5eVJki5cuKAzZ84oLy9Pzs7O6tWrlyRp0qRJuu+++/S73/1OQ4cO1R/+8Adt27ZNu3fvbvDxAQAAAABubXYN3SNHjtTZs2c1e/ZsWSwW9e7dW5mZmfL395ckWSyWas/s7tevn/Xn3NxcrV+/Xv7+/jp27JgkKTQ0VO+++66mTZum6dOnq1u3btq4caPuvvvuBhsXAAAAAACSnUO3JMXExCgmJqbG99LT06uVGYbxo20OHz5cw4cPv9GuAQAAAABwQ+y+ejkAAAAAAE0VoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACT2D10L1++XAEBAXJ1dVVQUJB27dp11fo7d+5UUFCQXF1d1bVrV61YsaJaneTkZPXo0UNubm7y9fXV5MmTdfHiRbOGAAAAAABAjewaujdu3Ki4uDglJibq4MGDCgsLU0REhAoKCmqsf/ToUQ0ZMkRhYWE6ePCgpk6dqtjYWGVkZFjrvPPOO5oyZYpmzpyp/Px8rV69Whs3blRCQkJDDQsAAAAAAElSc3sefNGiRRo/fryioqIkXZ6h3rJli1JSUjR//vxq9VesWCE/Pz8lJydLkgIDA7V//34tXLhQw4YNkyTl5OTonnvu0ejRoyVJXbp00ahRo7R3796GGRQAAAAAAP/HbjPdZWVlys3NVXh4uE15eHi49uzZU+M+OTk51eoPHjxY+/fvV3l5uSTp3nvvVW5urjVk//vf/1ZmZqYefvhhE0YBAAAAAEDt7DbTXVRUpIqKCnl7e9uUe3t7q7CwsMZ9CgsLa6x/6dIlFRUVycfHR7/4xS905swZ3XvvvTIMQ5cuXdLzzz+vKVOm1NqX0tJSlZaWWl+fO3fuBkYGAAAAAMBldl9IzcHBwea1YRjVyn6s/pXl2dnZmjt3rpYvX64DBw5o06ZN+uijjzRnzpxa25w/f748PDysm6+v7/UOBwAAAAAAK7vNdHt5ecnR0bHarPbp06erzWZX6dixY431mzdvrnbt2kmSpk+frsjISOt94n369NH333+vZ599VomJiWrWrPrfGRISEhQfH299fe7cOYI3AAAAAOCG2W2m29nZWUFBQcrKyrIpz8rKUmhoaI37hISEVKu/detWBQcHy8nJSZJUUlJSLVg7OjrKMAzrrPh/c3FxUevWrW02AAAAAABulF0vL4+Pj9ebb76ptLQ05efna/LkySooKFB0dLSkyzPQY8eOtdaPjo7W8ePHFR8fr/z8fKWlpWn16tV68cUXrXUeeeQRpaSk6N1339XRo0eVlZWl6dOn69FHH5Wjo2ODjxEAAAAAcOuy6yPDRo4cqbNnz2r27NmyWCzq3bu3MjMz5e/vL0myWCw2z+wOCAhQZmamJk+erGXLlqlTp05asmSJ9XFhkjRt2jQ5ODho2rRpOnnypNq3b69HHnlEc+fObfDxAQAAAABubXYN3ZIUExOjmJiYGt9LT0+vVjZgwAAdOHCg1vaaN2+umTNnaubMmfXVRQAAAAAArovdVy8HAAAAAKCpInQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACapc+g+ceKEvv76a+vrvXv3Ki4uTqtWrarXjgEAAAAA0NjVOXSPHj1aO3bskCQVFhZq0KBB2rt3r6ZOnarZs2fXewcBAAAAAGis6hy6P/vsM911112SpPfee0+9e/fWnj17tH79eqWnp9d3/wAAAAAAaLTqHLrLy8vl4uIiSdq2bZseffRRSVLPnj1lsVjqt3cAAAAAADRidQ7dt99+u1asWKFdu3YpKytLDz30kCTp1KlTateuXb13EAAAAACAxqrOoft3v/udVq5cqfvvv1+jRo3SHXfcIUn68MMPrZedAwAAAAAAqXldd7j//vtVVFSkc+fOydPT01r+7LPPqkWLFvXaOQAAAAAAGrPrek63YRjKzc3VypUrdf78eUmSs7MzoRsAAAAAgCvUeab7+PHjeuihh1RQUKDS0lINGjRIrVq1UlJSki5evKgVK1aY0U8AAAAAABqdOs90T5o0ScHBwfr222/l5uZmLX/88ce1ffv2eu0cAAAAAACNWZ1nunfv3q2//e1vcnZ2tin39/fXyZMn661jAAAAAAA0dnWe6a6srFRFRUW18q+//lqtWrWql04BAAAAANAU1Dl0Dxo0SMnJydbXDg4OunDhgmbOnKkhQ4bUZ98AAAAAAGjU6nx5+eLFi/XAAw+oV69eunjxokaPHq0jR47Iy8tLGzZsMKOPAAAAAAA0SnUO3Z06dVJeXp42bNigAwcOqLKyUuPHj9eYMWNsFlYDAAAAAOBWV+fQLUlubm4aN26cxo0bV9/9AQAAAACgyahz6F63bt1V3x87dux1dwYAAAAAgKakzqF70qRJNq/Ly8tVUlIiZ2dntWjRgtANAAAAAMD/qfPq5d9++63NduHCBX3xxRe69957WUgNAAAAAIAr1Dl01+S2227Tq6++Wm0WHAAAAACAW1m9hG5JcnR01KlTp+qrOQAAAAAAGr0639P94Ycf2rw2DEMWi0VLly7VPffcU28dAwAAAACgsatz6H7sscdsXjs4OKh9+/b63//9X7322mv11S8AAAAAABq9OofuyspKM/oBAAAAAECTU2/3dAMAAAAAAFvXNNMdHx9/zQ0uWrToujsDAAAAAEBTck2h++DBg9fUmIODww11BgAAAACApuSaQveOHTvM7gcAAAAAAE0O93QDAAAAAGCSOq9eLkn79u3T73//exUUFKisrMzmvU2bNtVLxwAAAAAAaOzqPNP97rvv6p577tHhw4e1efNmlZeX6/Dhw/rLX/4iDw8PM/oIAAAAAECjVOfQPW/ePC1evFgfffSRnJ2d9frrrys/P18jRoyQn5+fGX0EAAAAAKBRqnPo/uqrr/Twww9LklxcXPT999/LwcFBkydP1qpVq+q9gwAAAAAANFZ1Dt1t27bV+fPnJUmdO3fWZ599Jkn67rvvVFJSUr+9AwAAAACgEbvm0J2XlydJCgsLU1ZWliRpxIgRmjRpkiZMmKBRo0bpwQcfNKWTAAAAAAA0Rte8enn//v3Vr18/PfbYYxo1apQkKSEhQU5OTtq9e7eeeOIJTZ8+3bSOAgAAAADQ2FzzTPff/vY39e/fXwsXLlS3bt30y1/+Ujt37tRLL72kDz/8UIsWLZKnp6eZfQUAAAAAoFG55tAdEhKi1NRUFRYWKiUlRV9//bUGDhyobt26ae7cufr666/N7CcAAAAAAI1OnRdSc3Nz01NPPaXs7Gz961//0qhRo7Ry5UoFBARoyJAhZvQRAAAAAIBGqc6h+0rdunXTlClTlJiYqNatW2vLli311S8AAAAAABq9a15I7b/t3LlTaWlpysjIkKOjo0aMGKHx48fXZ98AAAAAAGjU6hS6T5w4ofT0dKWnp+vo0aMKDQ3VG2+8oREjRqhly5Zm9REAAAAAgEbpmkP3oEGDtGPHDrVv315jx47VuHHj1KNHDzP7BgAAAABAo3bNodvNzU0ZGRn6+c9/LkdHRzP7BAAAAABAk3DNofvDDz80sx8AAAAAADQ5N7R6OQAAAAAAqJ3dQ/fy5csVEBAgV1dXBQUFadeuXVetv3PnTgUFBcnV1VVdu3bVihUrqtX57rvvNHHiRPn4+MjV1VWBgYHKzMw0awgAAAAAANTIrqF748aNiouLU2Jiog4ePKiwsDBFRESooKCgxvpHjx7VkCFDFBYWpoMHD2rq1KmKjY1VRkaGtU5ZWZkGDRqkY8eO6f3339cXX3yh1NRUde7cuaGGBQAAAACApBt4Tnd9WLRokcaPH6+oqChJUnJysrZs2aKUlBTNnz+/Wv0VK1bIz89PycnJkqTAwEDt379fCxcu1LBhwyRJaWlp+uabb7Rnzx45OTlJkvz9/RtmQAAAAAAAXMFuM91lZWXKzc1VeHi4TXl4eLj27NlT4z45OTnV6g8ePFj79+9XeXm5pMsLvoWEhGjixIny9vZW7969NW/ePFVUVJgzEAAAAAAAamG3me6ioiJVVFTI29vbptzb21uFhYU17lNYWFhj/UuXLqmoqEg+Pj7697//rb/85S8aM2aMMjMzdeTIEU2cOFGXLl3SjBkzamy3tLRUpaWl1tfnzp27wdEBAAAAAHATLKTm4OBg89owjGplP1b/yvLKykp16NBBq1atUlBQkH7xi18oMTFRKSkptbY5f/58eXh4WDdfX9/rHQ4AAAAAAFZ2C91eXl5ydHSsNqt9+vTparPZVTp27Fhj/ebNm6tdu3aSJB8fH/30pz+Vo6OjtU5gYKAKCwtVVlZWY7sJCQkqLi62bidOnLiRoQEAAAAAIMmOodvZ2VlBQUHKysqyKc/KylJoaGiN+4SEhFSrv3XrVgUHB1sXTbvnnnv05ZdfqrKy0lrnX//6l3x8fOTs7Fxjuy4uLmrdurXNBgAAAADAjbLr5eXx8fF68803lZaWpvz8fE2ePFkFBQWKjo6WdHkGeuzYsdb60dHROn78uOLj45Wfn6+0tDStXr1aL774orXO888/r7Nnz2rSpEn617/+pT/96U+aN2+eJk6c2ODjAwAAAADc2uz6yLCRI0fq7Nmzmj17tiwWi3r37q3MzEzrI74sFovNM7sDAgKUmZmpyZMna9myZerUqZOWLFlifVyYJPn6+mrr1q2aPHmy+vbtq86dO2vSpEl6+eWXG3x8AAAAAIBbm11DtyTFxMQoJiamxvfS09OrlQ0YMEAHDhy4apshISH6+9//Xh/dAwAAAADgutl99XIAAAAAAJoqQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASu4fu5cuXKyAgQK6urgoKCtKuXbuuWn/nzp0KCgqSq6urunbtqhUrVtRa991335WDg4Mee+yxeu41AAAAAAA/zq6he+PGjYqLi1NiYqIOHjyosLAwRUREqKCgoMb6R48e1ZAhQxQWFqaDBw9q6tSpio2NVUZGRrW6x48f14svvqiwsDCzhwEAAAAAQI3sGroXLVqk8ePHKyoqSoGBgUpOTpavr69SUlJqrL9ixQr5+fkpOTlZgYGBioqK0rhx47Rw4UKbehUVFRozZox++9vfqmvXrg0xFAAAAAAAqrFb6C4rK1Nubq7Cw8NtysPDw7Vnz54a98nJyalWf/Dgwdq/f7/Ky8utZbNnz1b79u01fvz4a+pLaWmpzp07Z7MBAAAAAHCj7Ba6i4qKVFFRIW9vb5tyb29vFRYW1rhPYWFhjfUvXbqkoqIiSdLf/vY3rV69Wqmpqdfcl/nz58vDw8O6+fr61nE0AAAAAABUZ/eF1BwcHGxeG4ZRrezH6leVnz9/Xr/85S+VmpoqLy+va+5DQkKCiouLrduJEyfqMAIAAAAAAGrW3F4H9vLykqOjY7VZ7dOnT1ebza7SsWPHGus3b95c7dq10+eff65jx47pkUcesb5fWVkpSWrevLm++OILdevWrVq7Li4ucnFxudEhAQAAAABgw24z3c7OzgoKClJWVpZNeVZWlkJDQ2vcJyQkpFr9rVu3Kjg4WE5OTurZs6cOHTqkvLw86/boo4/qgQceUF5eHpeNAwAAAAAalN1muiUpPj5ekZGRCg4OVkhIiFatWqWCggJFR0dLunzZ98mTJ7Vu3TpJUnR0tJYuXar4+HhNmDBBOTk5Wr16tTZs2CBJcnV1Ve/evW2O0aZNG0mqVg4AAAAAgNnsGrpHjhyps2fPavbs2bJYLOrdu7cyMzPl7+8vSbJYLDbP7A4ICFBmZqYmT56sZcuWqVOnTlqyZImGDRtmryEAAAAAAFAru4ZuSYqJiVFMTEyN76Wnp1crGzBggA4cOHDN7dfUBgAAAAAADcHuq5cDAAAAANBUEboBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAADVavny5AgIC5OrqqqCgIO3ateuq9Xfu3KmgoCC5urqqa9euWrFihc37qampCgsLk6enpzw9PTVw4EDt3bvXps758+cVFxcnf39/ubm5KTQ0VPv27av3sQFAQyF0AwAAoJqNGzcqLi5OiYmJOnjwoMLCwhQREaGCgoIa6x89elRDhgxRWFiYDh48qKlTpyo2NlYZGRnWOtnZ2Ro1apR27NihnJwc+fn5KTw8XCdPnrTWiYqKUlZWlt566y0dOnRI4eHhGjhwoE0dAGhMCN0AAACoZtGiRRo/fryioqIUGBio5ORk+fr6KiUlpcb6K1askJ+fn5KTkxUYGKioqCiNGzdOCxcutNZ55513FBMTozvvvFM9e/ZUamqqKisrtX37dknSDz/8oIyMDCUlJem+++5T9+7dNWvWLAUEBNR6XAC42RG6AQAAYKOsrEy5ubkKDw+3KQ8PD9eePXtq3CcnJ6da/cGDB2v//v0qLy+vcZ+SkhKVl5erbdu2kqRLly6poqJCrq6uNvXc3Ny0e/fu6x0OANgVoRsAAAA2vvnmG1VUVMjb29um3NvbW4WFhTXuU1hYWGP9S5cuqaioqMZ9pkyZos6dO2vgwIGSpFatWikkJERz5szRqVOnVFFRobfffluffPKJLBZLPYwMABoeoRsAAAA1cnBwsHltGEa1sh+rX1O5JCUlJWnDhg3atGmTzcz2W2+9JcMw1LlzZ7m4uGjJkiUaPXq0HB0db2QoAGA3hG4AAADYaNu2rRwdHavNap8+fbrabHaVjh071li/efPmateunU35woULNW/ePG3dulV9+/a1ea9bt27auXOnLly4oBMnTmjv3r0qLy9XQEBAPYwMABoeoRs3DXs8lqRLly5ycHCotk2cOLHexwfcKjiXgcbP2dlZQUFBysrKsinPyspSaGhojfuEhIRUq79161YFBwfLycnJWrZgwQLNmTNHH3/8sYKDg2vtQ8uWLeXj46Nvv/1WW7Zs0dChQ29gRABgP4Ru3BTs9ViSffv2yWKxWLeqLwtPPvmkuQMGmqg//OEPnMtAExEfH68333xTaWlpys/P1+TJk1VQUKDo6GhJUkJCgsaOHWutHx0drePHjys+Pl75+flKS0vT6tWr9eKLL1rrJCUladq0aUpLS1OXLl1UWFiowsJCXbhwwVpny5Yt+vjjj3X06FFlZWXpgQceUI8ePfTMM8803OABoD4ZqKa4uNiQZBQXF9u7K1d16tQpY9asWYaPzylDMhrd5uNzuf+nTp0y7rrrLiM6OtpmfD179jSmTJlS49hfeuklo2fPnjZlzz33nPGzn/2s1s/r0qVLRqtWrYy1a9fWWmfSpElGt27djMrKyjr8lwBuTGM/l688n/v168e5jFtWUzqXT506ZRiGYSxbtszw9/c3nJ2djf79+xs7d+60jvepp54yBgwYYPMZZGdnG/369TOcnZ2NLl26GCkpKTbv+/v7G5KqbTNnzrTW2bhxo9G1a1fD2dnZ6NixozFx4kTju+++M+2/G/DfmuK5DHNca25kpht2Z6/HktTUj7ffflvjxo276iIxAGp26dIlffrpp5zLQBMSExOjY8eOqbS0VLm5ubrvvvus76Wnpys7O9um/oABA3TgwAGVlpbq6NGj1lnxKseOHZNhGNW2WbNmWeuMGDFCX331lUpLS2WxWLR06VJ5eHiYOUygyePWL/sidMPu7PVYkv/2wQcf6LvvvtPTTz9d90EAUElJCecyAAA3GW7jtL/m9u4AUKUhHkuSnZ1t81iSK61evVoRERHq1KlTXbsO4AqcywAA3DwWLVqk8ePHKyoqSpKUnJysLVu2KCUlRfPnz69Wf8WKFfLz81NycrIkKTAwUPv379fChQs1bNgwSdI777xjs09qaqref/99bd++3brWQ/v27W3qvPrqq+rWrZsGDBhQ30O86RG6YXcN9ViSbdu2VXssSZXjx49r27Zt2rRp0w2MBLi1tWjRgnMZaCJqu9KkMWjRogWXowP/p+o2zilTptiUX8+tX6tXr1Z5ebnN0wiqXOutX/Hx8bfkrV+EbtjdlY8lefzxx63lWVlZtT4eJCQkRH/84x9tymp7LMkrr7yiLVu2XPWxJGvWrFGHDh308MMP3+BogFtX8+bN1bdvX85loBFzd7+gSlU26j9cNWveTLEvxBK8AZlzG6ePj0+1fbj16+oI3bgpxMfHKzIyUsHBwQoJCdGqVauqPZbk5MmTWrdunaTLjyVZunSp4uPjNWHCBOXk5Gj16tXasGGDtc2kpCRNnz5d69evtz6WRJLc3d3l7u5urVdZWak1a9boqaeeUvPmnBLAjXj22WcVGxvLuQw0Uq6uF9VMzZShDBWp8c12e8lLwy4NU0lJCaEbuAK3ftkX30pwUxg5cqTOnj2r2bNny2KxqHfv3srMzJS/v78kyWKx2Cz2EBAQoMzMTE2ePFnLli1Tp06dtGTJEut9JtLlVRrLyso0fPhwm2PNnDnTZpXUbdu2qaCgQOPGjTN3kMAtYOjQoaqsrORcBhq5IhXJIou9uwHgBnEb582B0I2bRkxMjGJiYmp8Lz09vVpZ1WNJanPs2LFrOm54eLj1r3cAbhznMgAANwdu47w58MgwAAAAAGii4uPj9eabbyotLU35+fmaPHlytVu/qlYcly7f+nX8+HHFx8crPz9faWlpWr16tV588UVrnaSkJE2bNk1paWnWW78KCwt14cIFm2Nz69dlt+7IAQAAAKCJ4zZO+yN0w+4a82NJJB5NAlypMZ/PnMsAgKaKW7/si9ANu2kKjyWReDQJIDWN85lzGQAAmIHQDbtp7I8lkXg0CVClsZ/PnMsAAMAshG7YHY8lAZoOzmcAAG4ejfm2L6np3PpF6AYAAACAJqQp3PYlNZ1bvwjdAAAAANCENPbbvqSmdesXoRsAAAAAmiBu+7o5NLN3BwAAAAAAaKoI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJjE7qF7+fLlCggIkKurq4KCgrRr166r1t+5c6eCgoLk6uqqrl27asWKFTbvp6amKiwsTJ6envL09NTAgQO1d+9eM4cAAAAAAECN7Bq6N27cqLi4OCUmJurgwYMKCwtTRESECgoKaqx/9OhRDRkyRGFhYTp48KCmTp2q2NhYZWRkWOtkZ2dr1KhR2rFjh3JycuTn56fw8HCdPHmyoYYFAAAAAIAkO4fuRYsWafz48YqKilJgYKCSk5Pl6+urlJSUGuuvWLFCfn5+Sk5OVmBgoKKiojRu3DgtXLjQWuedd95RTEyM7rzzTvXs2VOpqamqrKzU9u3bG2pYAAAAAABIsmPoLisrU25ursLDw23Kw8PDtWfPnhr3ycnJqVZ/8ODB2r9/v8rLy2vcp6SkROXl5Wrbtm2tfSktLdW5c+dsNgAAAAAAbpTdQndRUZEqKirk7e1tU+7t7a3CwsIa9yksLKyx/qVLl1RUVFTjPlOmTFHnzp01cODAWvsyf/58eXh4WDdfX986jgYAAAAAgOrsvpCag4ODzWvDMKqV/Vj9msolKSkpSRs2bNCmTZvk6upaa5sJCQkqLi62bidOnKjLEAAAAAAAqFFzex3Yy8tLjo6O1Wa1T58+XW02u0rHjh1rrN+8eXO1a9fOpnzhwoWaN2+etm3bpr59+161Ly4uLnJxcbmOUQAAAAAAUDu7zXQ7OzsrKChIWVlZNuVZWVkKDQ2tcZ+QkJBq9bdu3arg4GA5OTlZyxYsWKA5c+bo448/VnBwcP13HgAAAACAa2DXy8vj4+P15ptvKi0tTfn5+Zo8ebIKCgoUHR0t6fJl32PHjrXWj46O1vHjxxUfH6/8/HylpaVp9erVevHFF611kpKSNG3aNKWlpalLly4qLCxUYWGhLly40ODjAwAAAADc2ux2ebkkjRw5UmfPntXs2bNlsVjUu3dvZWZmyt/fX5JksVhsntkdEBCgzMxMTZ48WcuWLVOnTp20ZMkSDRs2zFpn+fLlKisr0/Dhw22ONXPmTM2aNatBxgUAAAAAgGTn0C1JMTExiomJqfG99PT0amUDBgzQgQMHam3v2LFj9dQzAAAAAABujN1XLwcAAAAAoKkidAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASu4fu5cuXKyAgQK6urgoKCtKuXbuuWn/nzp0KCgqSq6urunbtqhUrVlSrk5GRoV69esnFxUW9evXS5s2bzeo+AAAAAAC1smvo3rhxo+Li4pSYmKiDBw8qLCxMERERKigoqLH+0aNHNWTIEIWFhengwYOaOnWqYmNjlZGRYa2Tk5OjkSNHKjIyUv/4xz8UGRmpESNG6JNPPmmoYQEAAAAAIMnOoXvRokUaP368oqKiFBgYqOTkZPn6+iolJaXG+itWrJCfn5+Sk5MVGBioqKgojRs3TgsXLrTWSU5O1qBBg5SQkKCePXsqISFBDz74oJKTkxtoVAAAAAAAXGa30F1WVqbc3FyFh4fblIeHh2vPnj017pOTk1Ot/uDBg7V//36Vl5dftU5tbQIAAAAAYJbm9jpwUVGRKioq5O3tbVPu7e2twsLCGvcpLCyssf6lS5dUVFQkHx+fWuvU1qYklZaWqrS01Pq6uLhYknTu3Lk6jamhnT9/XhcvXpSn51FVVp63d3fqzN3968v9l6cqVWnv7lwXT3nqoi7q/Pnzatmypb27g0aqsZ/LUuM/nzmXUR84l+2Pcxn1gXP55tAYzueqvGgYxtUrGnZy8uRJQ5KxZ88em/JXXnnF6NGjR4373Hbbbca8efNsynbv3m1IMiwWi2EYhuHk5GSsX7/eps7bb79tuLi41NqXmTNnGpLY2NjY2NjY2NjY2NjY2Oq0nThx4qrZ124z3V5eXnJ0dKw2A3369OlqM9VVOnbsWGP95s2bq127dletU1ubkpSQkKD4+Hjr68rKSn3zzTdq166dHBwc6jQuXLtz587J19dXJ06cUOvWre3dHQA3gPMZaBo4l4GmgXO5YRiGofPnz6tTp05XrWe30O3s7KygoCBlZWXp8ccft5ZnZWVp6NChNe4TEhKiP/7xjzZlW7duVXBwsJycnKx1srKyNHnyZJs6oaGhtfbFxcVFLi4uNmVt2rSp65BwnVq3bs0vA6CJ4HwGmgbOZaBp4Fw2n4eHx4/WsVvolqT4+HhFRkYqODhYISEhWrVqlQoKChQdHS3p8gz0yZMntW7dOklSdHS0li5dqvj4eE2YMEE5OTlavXq1NmzYYG1z0qRJuu+++/S73/1OQ4cO1R/+8Adt27ZNu3fvtssYAQAAAAC3LruG7pEjR+rs2bOaPXu2LBaLevfurczMTPn7+0uSLBaLzTO7AwIClJmZqcmTJ2vZsmXq1KmTlixZomHDhlnrhIaG6t1339W0adM0ffp0devWTRs3btTdd9/d4OMDAAAAANzaHAzjx5ZaA8xRWlqq+fPnKyEhodrl/QAaF85noGngXAaaBs7lmwuhGwAAAAAAkzSzdwcAAAAAAGiqCN0AAAAAAJiE0A0AqFfp6ek8dhH4L/fff7/i4uLs3Y1aFRYWatCgQWrZsiXnL9CIHTt2TA4ODsrLy7N3V3AFQjfq5PTp03ruuefk5+cnFxcXdezYUYMHD1ZOTo61zp49ezRkyBB5enrK1dVVffr00WuvvaaKigqbthwcHKpt9957r55++uka37tyu9a+ALeyHzuPnn76aUnSjh079MADD6ht27Zq0aKFbrvtNj311FO6dOmSJCk7O1sODg7y9PTUxYsXbY6xd+9em/NSuvxkin/9618NNk6gMdi0aZPGjx//o+flrFmz7NK/xYsXy2KxKC8vj/MXqEdXfq91cnJS165d9eKLL+r777//0X2r/v397rvvTOnP9X7PrurX1bb09PR663NTYNdHhqHxGTZsmMrLy7V27Vp17dpV//nPf7R9+3Z98803kqTNmzdrxIgReuaZZ7Rjxw61adNG27Zt00svvaS///3veu+992y+nK9Zs0YPPfSQ9bWzs7McHR316quvWst8fHyq1buWvgC3OovFYv1548aNmjFjhr744gtrmZubmz7//HNFREQoNjZWb7zxhtzc3HTkyBG9//77qqystGmvVatW2rx5s0aNGmUtS0tLk5+fn83jHd3c3OTm5mbiyKTy8nI5OTmZegygPrVt21YeHh425+XChQv18ccfa9u2bdYyd3d368+GYaiiokLNm5v/de2rr75SUFCQbrvttutuo6HPS34PoLF46KGHtGbNGpWXl2vXrl2KiorS999/r5SUlAbvy+uvv37D37MHDRpk87ts0qRJOnfunNasWWMt8/DwMH8wjYkBXKNvv/3WkGRkZ2fX+P6FCxeMdu3aGU888US19z788ENDkvHuu+9ayyQZmzdv/tHj1lTvx/oCwNaaNWsMDw+PauWLFy82unTpctV9d+zYYUgypk2bZgwcONBaXlJSYnh4eBjTp083rvzn5MpjVVZWGg8++KAxePBgo7Ky0jCMy+evr6+vMXXqVOs+aWlpRs+ePQ0XFxejR48exrJly6zvHT161JBkbNy40RgwYIDh4uJipKWlXc/HANjNgAEDjEmTJtmUzZw507jjjjusr6vOtY8//tgICgoynJycjL/85S/Gl19+aTz66KNGhw4djJYtWxrBwcFGVlaWTVv+/v7G3LlzjWeeecZwd3c3fH19jZUrV1rfLy0tNSZOnGh07NjRcHFxMfz9/Y158+ZZ95Vk3Z566inDMAzj+PHjxqOPPmq0bNnSaNWqlfHkk08ahYWF1fq/evVqIyAgwHBwcDAqKysNScaKFSuMhx9+2HBzczN69uxp7Nmzxzhy5IgxYMAAo0WLFsbPfvYz48svv7QZw4cffmj079/fcHFxMQICAoxZs2YZ5eXl1vclGSkpKcajjz5qtGjRwpgxY8aN/CcBGsRTTz1lDB061KYsKirK6Nixo/HWW28ZQUFBhru7u+Ht7W2MGjXK+M9//mMYxv//t6+mc7OiosJ49dVXjW7duhnOzs6Gr6+v8corr9jsl5GRYdx///2Gm5ub0bdvX2PPnj019q8+vmfXNEbY4vJyXDN3d3e5u7vrgw8+UGlpabX3t27dqrNnz+rFF1+s9t4jjzyin/70p9qwYUOD9AXAtenYsaMsFov++te//mjdyMhI7dq1yzqrnZGRoS5duqh///617uPg4KC1a9dq7969WrJkiSQpOjpa3t7e1stoU1NTlZiYqLlz5yo/P1/z5s3T9OnTtXbtWpu2Xn75ZcXGxio/P1+DBw++zhEDN7+XXnpJ8+fPV35+vvr27asLFy5oyJAh2rZtmw4ePKjBgwfrkUcesbnCRJJee+01BQcH6+DBg4qJidHzzz+vf/7zn5KkJUuW6MMPP9R7772nL774Qm+//ba6dOkiSdq3b58eeughjRgxQhaLRa+//roMw9Bjjz2mb775Rjt37lRWVpa++uorjRw50uaYX375pd577z1lZGTY3EM6Z84cjR07Vnl5eerZs6dGjx6t5557TgkJCdq/f78k6YUXXrDW37Jli375y18qNjZWhw8f1sqVK5Wenq65c+faHG/mzJkaOnSoDh06pHHjxtXXRw40KDc3N5WXl6usrExz5szRP/7xD33wwQc6evSo9dYvX19fZWRkSJK++OIL67kpSQkJCfrd736n6dOn6/Dhw1q/fr28vb1tjpGYmKgXX3xReXl5+ulPf6pRo0ZZbxv7MXzPNoG9Uz8al/fff9/w9PQ0XF1djdDQUCMhIcH4xz/+YRiGYbz66quGJOPbb7+tcd9HH33UCAwMtL6WZLi6uhotW7a0bjXNfKuWGfGr9QWArdpmui9dumQ8/fTThiSjY8eOxmOPPWa88cYbRnFxsbVO1ezbt99+azz22GPGb3/7W8MwDOOBBx4wXn/9dWPz5s21znRXee+99wwXFxcjISHBaNGihfHFF19Y3/P19TXWr19vU3/OnDlGSEiIYRj//6/2ycnJN/oxAHZTl5nuDz744Efb69Wrl/HGG29YX/v7+xu//OUvra8rKyuNDh06GCkpKYZhGMavfvUr43//93+tV5z8t6FDh1pn0QzDMLZu3Wo4OjoaBQUF1rLPP//ckGTs3bvX2n8nJyfj9OnTNm3p/66MqZKTk2NIMlavXm0t27Bhg+Hq6mp9HRYWZp15r/LWW28ZPj4+Nu3GxcXV/qEAN6H/ngX+5JNPjHbt2hkjRoyoVnfv3r2GJOP8+fOGYdj++1vl3LlzhouLi5Gamlrj8ar+zXzzzTetZVXnbn5+frX69fE9m5nuH8dMN+pk2LBhOnXqlD788EMNHjxY2dnZ6t+/v81iCYZh1LivYRg293NLlxduycvLs26DBg2q174AuDpHR0etWbNGX3/9tZKSktSpUyfNnTtXt99+u839WlXGjRun9PR0/fvf/1ZOTo7GjBlzTcd58skn9cQTT2j+/Pl67bXX9NOf/lSSdObMGZ04cULjx4+3/mXd3d1dr7zyir766iubNoKDg298wEAj8N//r3///fd66aWX1KtXL7Vp00bu7u765z//WW2mu2/fvtafHRwc1LFjR50+fVrS5cWT8vLy1KNHD8XGxmrr1q1X7UN+fr58fX3l6+trLas6fn5+vrXM399f7du3r7b/lX2pmoHr06ePTdnFixd17tw5SVJubq5mz55t83tgwoQJslgsKikpqfWzARqDjz76SO7u7nJ1dVVISIjuu+8+vfHGGzp48KCGDh0qf39/tWrVSvfff78kVTu3r5Sfn6/S0lI9+OCDVz3mleegj4+PJFl/H1wLvmfXL0I36szV1VWDBg3SjBkztGfPHj399NOaOXOm9Uv0lf8YX+mf//xntQVaOnbsqO7du1u3li1b1ktfANRN586dFRkZqWXLlunw4cO6ePGiVqxYUa3ekCFDdPHiRY0fP16PPPKI2rVrd03tl5SUKDc3V46Ojjpy5Ii1vGqxttTUVJs/wH322Wf6+9//btNGXX8/AI3Vf/+//pvf/EYZGRmaO3eudu3apby8PPXp00dlZWU29f57UTEHBwfrOda/f38dPXpUc+bM0Q8//KARI0Zo+PDhtfahpj+U11Re23l5ZV+q6tdUVtW/yspK/fa3v7X5PXDo0CEdOXJErq6uP3o84Gb2wAMPKC8vT1988YUuXryoTZs2qWXLlgoPD5e7u7vefvtt7du3T5s3b5akauf2la51odKrnW/Xiu/Z9YfQjRvWq1cvff/99woPD1fbtm312muvVavz4Ycf6siRIzarHpvZFwDXz9PTUz4+PjWeS46OjoqMjFR2dnad7qf89a9/rWbNmunPf/6zlixZor/85S+SLs92de7cWf/+979t/gDXvXt3BQQE1NuYgMZs165devrpp/X444+rT58+6tixo44dO1bndlq3bq2RI0cqNTVVGzduVEZGRq1P/OjVq5cKCgp04sQJa9nhw4dVXFyswMDA6x1Krfr3768vvvii2u+B7t27q1kzvq6icWvZsqW6d+8uf39/axj+5z//qaKiIr366qsKCwtTz549q81EOzs7S5LNY3dvu+02ubm5afv27Q03gP/D9+zrxyPDcM3Onj2rJ598UuPGjVPfvn3VqlUr7d+/X0lJSRo6dKhatmyplStX6he/+IWeffZZvfDCC2rdurW2b9+u3/zmNxo+fLhGjBjRIH0BcG1WrlypvLw8Pf744+rWrZsuXryodevW6fPPP9cbb7xR4z5z5szRb37zm2ue5f7Tn/6ktLQ05eTkqH///poyZYqeeuopffrpp/L09NSsWbMUGxur1q1bKyIiQqWlpdq/f7++/fZbxcfH1+dwgUape/fu2rRpkx555BE5ODho+vTpdZ6xWrx4sXx8fHTnnXeqWbNm+v3vf6+OHTuqTZs2NdYfOHCg+vbtqzFjxig5OVmXLl1STEyMBgwYYMol3jNmzNDPf/5z+fr66sknn1SzZs306aef6tChQ3rllVfq/XiAvfn5+cnZ2VlvvPGGoqOj9dlnn2nOnDk2dfz9/eXg4KCPPvpIQ4YMkZubm9zd3fXyyy/rpZdekrOzs+655x6dOXNGn3/+ucaPH18vfeN7dv0jdOOaubu76+6779bixYv11Vdfqby8XL6+vpowYYKmTp0qSRo+fLh27NihefPm6b777tMPP/yg7t27KzExUXFxcTVeqmZWXwD8uLvuuku7d+9WdHS0Tp06JXd3d91+++364IMPNGDAgBr3cXZ2lpeX1zW1f+bMGY0fP16zZs2yrnI+c+ZMbd26VdHR0dq4caOioqLUokULLViwQC+99JJatmypPn36KC4urr6GCTRqixcv1rhx4xQaGiovLy+9/PLL1nuhr5W7u7t+97vf6ciRI3J0dNT//M//KDMzs9ZZZAcHB33wwQf61a9+pfvuu0/NmjXTQw89VOsf427U4MGD9dFHH2n27NlKSkqSk5OTevbsqaioKFOOB9hb+/btlZ6erqlTp2rJkiXq37+/Fi5cqEcffdRap3Pnzvrtb3+rKVOm6JlnntHYsWOVnp6u6dOnq3nz5poxY4ZOnTolHx8fRUdH11vf+J5d/xyM2la9AgAAAAAAN4SbZAAAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQDANcvOzpaDg4O+++67a96nS5cuSk5ONq1PAADczAjdAAA0IU8//bQcHBwUHR1d7b2YmBg5ODjo6aefbviOAQBwiyJ0AwDQxPj6+urdd9/VDz/8YC27ePGiNmzYID8/Pzv2DACAWw+hGwCAJqZ///7y8/PTpk2brGWbNm2Sr6+v+vXrZy0rLS1VbGysOnToIFdXV917773at2+fTVuZmZn66U9/Kjc3Nz3wwAM6duxYtePt2bNH9913n9zc3OTr66vY2Fh9//33tfZv1qxZ8vPzk4uLizp16qTY2NgbHzQAADcpQjcAAE3QM888ozVr1lhfp6Wlady4cTZ1XnrpJWVkZGjt2rU6cOCAunfvrsGDB+ubb76RJJ04cUJPPPGEhgwZory8PEVFRWnKlCk2bRw6dEiDBw/WE088oU8//VQbN27U7t279cILL9TYr/fff1+LFy/WypUrdeTIEX3wwQfq06dPPY8eAICbB6EbAIAmKDIyUrt379axY8d0/Phx/e1vf9Mvf/lL6/vff/+9UlJStGDBAkVERKhXr15KTU2Vm5ubVq9eLUlKSUlR165dtXjxYvXo0UNjxoypdj/4ggULNHr0aMXFxem2225TaGiolixZonXr1unixYvV+lVQUKCOHTtq4MCB8vPz01133aUJEyaY+lkAAGBPhG4AAJogLy8vPfzww1q7dq3WrFmjhx9+WF5eXtb3v/rqK5WXl+uee+6xljk5Oemuu+5Sfn6+JCk/P18/+9nP5ODgYK0TEhJic5zc3Fylp6fL3d3dug0ePFiVlZU6evRotX49+eST+uGHH9S1a1dNmDBBmzdv1qVLl+p7+AAA3DSa27sDAADAHOPGjbNe5r1s2TKb9wzDkCSbQF1VXlVWVedqKisr9dxzz9V4X3ZNi7b5+vrqiy++UFZWlrZt26aYmBgtWLBAO3fulJOT07UNDACARoSZbgAAmqiHHnpIZWVlKisr0+DBg23e6969u5ydnbV7925rWXl5ufbv36/AwEBJUq9evfT3v//dZr//ft2/f399/vnn6t69e7XN2dm5xn65ubnp0Ucf1ZIlS5Sdna2cnBwdOnSoPoYMAMBNh5luAACaKEdHR+ul4o6OjjbvtWzZUs8//7x+85vfqG3btvLz81NSUpJKSko0fvx4SVJ0dLRee+01xcfH67nnnrNeSn6ll19+WT/72c80ceJETZgwQS1btlR+fr6ysrL0xhtvVOtTenq6KioqdPfdd6tFixZ666235ObmJn9/f3M+BAAA7IyZbgAAmrDWrVurdevWNb736quvatiwYYqMjFT//v315ZdfasuWLfL09JR0+fLwjIwM/fGPf9Qdd9yhFStWaN68eTZt9O3bVzt37tSRI0cUFhamfv36afr06fLx8anxmG3atFFqaqruuece9e3bV9u3b9cf//hHtWvXrn4HDgDATcLBuJYbtgAAAAAAQJ0x0w0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJjk/wEDT8PNWPm/5QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics_df = pd.read_csv('ettm1_results.csv')\n",
    "\n",
    "metrics = metrics_df['metric']\n",
    "models = metrics_df.columns[2:]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "bar_width = 0.2\n",
    "\n",
    "r1 = range(len(models))\n",
    "r2 = [x + bar_width for x in r1]\n",
    "\n",
    "ax.bar(r1, metrics_df[metrics_df['metric'] == 'mae'].values[0][2:], color='b', width=bar_width, edgecolor='grey', label='mae')\n",
    "ax.bar(r2, metrics_df[metrics_df['metric'] == 'mse'].values[0][2:], color='g', width=bar_width, edgecolor='grey', label='mse')\n",
    "\n",
    "for i, v in enumerate(metrics_df[metrics_df['metric'] == 'mae'].values[0][2:]):\n",
    "    ax.text(i - 0.1, v + 0.001, str(round(v, 3)))\n",
    "for i, v in enumerate(metrics_df[metrics_df['metric'] == 'mse'].values[0][2:]):\n",
    "    ax.text(i + bar_width - 0.1, v + 0.001, str(round(v, 3)))\n",
    "\n",
    "ax.set_xlabel('Models')\n",
    "ax.set_ylabel('Values')\n",
    "\n",
    "ax.set_xticks([r + bar_width/2 for r in range(len(models))])\n",
    "ax.set_xticklabels(models)\n",
    "\n",
    "ax.legend()\n",
    "\n",
    "plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multivariate forecasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_df, val_size, test_size, freq = load_data('ettm2')\n",
    "\n",
    "horizon = 96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n",
      "INFO:lightning_fabric.utilities.seed:Seed set to 1\n"
     ]
    }
   ],
   "source": [
    "models = [SOFTS(h=horizon, input_size=3*horizon, n_series=7, max_steps=1000, early_stop_patience_steps=3, scaler_type='identity', valid_loss=MAE()),\n",
    "          TSMixer(h=horizon, input_size=3*horizon, n_series=7, max_steps=1000, early_stop_patience_steps=3, scaler_type='identity', valid_loss=MAE()),\n",
    "          iTransformer(h=horizon, input_size=3*horizon, n_series=7, max_steps=1000, early_stop_patience_steps=3, scaler_type='identity', valid_loss=MAE())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                   | Params\n",
      "---------------------------------------------------------\n",
      "0 | loss          | MAE                    | 0     \n",
      "1 | valid_loss    | MAE                    | 0     \n",
      "2 | padder        | ConstantPad1d          | 0     \n",
      "3 | scaler        | TemporalNorm           | 0     \n",
      "4 | enc_embedding | DataEmbedding_inverted | 147 K \n",
      "5 | encoder       | TransEncoder           | 6.8 M \n",
      "6 | projection    | Linear                 | 49.2 K\n",
      "---------------------------------------------------------\n",
      "7.0 M     Trainable params\n",
      "0         Non-trainable params\n",
      "7.0 M     Total params\n",
      "28.105    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a3aef21ebfc46bf8b8a93311911e56a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01f1f2c8556447a38f4cff550e51e6fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "483becf65c1f4055a55c39dbc2208057",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ca3e7fad52a43c6b336bef92d0ca12b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2488c98c8f8498c9eb06d0c62e1bcf7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d0cd13a18b44857a61dacb782b85261",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49201657478f4622998d05f2d842944e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                     | Params\n",
      "-----------------------------------------------------------\n",
      "0 | loss          | MAE                      | 0     \n",
      "1 | valid_loss    | MAE                      | 0     \n",
      "2 | padder        | ConstantPad1d            | 0     \n",
      "3 | scaler        | TemporalNorm             | 0     \n",
      "4 | norm          | ReversibleInstanceNorm1d | 14    \n",
      "5 | mixing_layers | Sequential               | 184 K \n",
      "6 | out           | Linear                   | 27.7 K\n",
      "-----------------------------------------------------------\n",
      "212 K     Trainable params\n",
      "0         Non-trainable params\n",
      "212 K     Total params\n",
      "0.849     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "930fe67acbc8461ebc71b92a75f5350b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62be6bd0cbfc4af4a4a040fb0e03263f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14ddd7c0b6df4f3091669b45efc2f8b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9fd82068c804e4b8a3356d1182a19d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12620a0b78344c09bbda2f05bb4e5a9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "682113d420d64e939040acb8708e0ab7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bea2ac3ef3e49808a7cc97717e83a2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2c75e4f7acc4924a46290ee344784f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2f663c50049426ebf15faa0162e392b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0aee5346ce5045eaa247d6349fca52dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54634b2a7b9c46059d5169fd20fd4dc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "700db3cc1a10485fabff2f016d87e568",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:`Trainer.fit` stopped: `max_steps=1000` reached.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7dc3513138da426cb5f28de6ee73d950",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
      "INFO:pytorch_lightning.callbacks.model_summary:\n",
      "  | Name          | Type                   | Params\n",
      "---------------------------------------------------------\n",
      "0 | loss          | MAE                    | 0     \n",
      "1 | valid_loss    | MAE                    | 0     \n",
      "2 | padder        | ConstantPad1d          | 0     \n",
      "3 | scaler        | TemporalNorm           | 0     \n",
      "4 | enc_embedding | DataEmbedding_inverted | 147 K \n",
      "5 | encoder       | TransEncoder           | 6.3 M \n",
      "6 | projector     | Linear                 | 49.2 K\n",
      "---------------------------------------------------------\n",
      "6.5 M     Trainable params\n",
      "0         Non-trainable params\n",
      "6.5 M     Total params\n",
      "26.012    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a9037ce3038482499f79f79a2199cfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49a7ebc2f6c24f4a83b6f4b30a4c05f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b544715d8ae40079145de5a263ce3fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa1b92460c744d68bce85a2ae9cd6065",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cdbe7912df64ead943c2ae17a732d03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41305f734fa24346a7934fd5f24f281c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_lightning.utilities.rank_zero:Trainer already configured with model summary callbacks: [<class 'pytorch_lightning.callbacks.model_summary.ModelSummary'>]. Skipping setting a default `ModelSummary` callback.\n",
      "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (mps), used: True\n",
      "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
      "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
      "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c86a93f1ba444419985c59fa9817fec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/marcopeix/dev/neuralforecast/neuralforecast/core.py:199: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "nf = NeuralForecast(models=models, freq='15min')\n",
    "\n",
    "nf_preds = nf.cross_validation(df=Y_df, val_size=val_size, test_size=test_size, n_windows=None)\n",
    "nf_preds = nf_preds.reset_index()\n",
    "\n",
    "ettm2_evaluation = evaluate(df=nf_preds, metrics=[mae, mse], models=['SOFTS', 'TSMixer', 'iTransformer'])\n",
    "ettm2_evaluation.to_csv('ettm2_results.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>SOFTS</th>\n",
       "      <th>TSMixer</th>\n",
       "      <th>iTransformer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mae</td>\n",
       "      <td>0.328205</td>\n",
       "      <td>0.248595</td>\n",
       "      <td>0.338629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mse</td>\n",
       "      <td>0.253429</td>\n",
       "      <td>0.166560</td>\n",
       "      <td>0.270181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  metric     SOFTS   TSMixer  iTransformer\n",
       "0    mae  0.328205  0.248595      0.338629\n",
       "1    mse  0.253429  0.166560      0.270181"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ettm2_metrics = pd.read_csv(\"ettm2_results.csv\")\n",
    "\n",
    "ettm2_metrics['SOFTS'] = pd.to_numeric(ettm2_metrics['SOFTS'], errors='coerce')\n",
    "ettm2_metrics['TSMixer'] = pd.to_numeric(ettm2_metrics['TSMixer'], errors='coerce')\n",
    "ettm2_metrics['iTransformer'] = pd.to_numeric(ettm2_metrics['iTransformer'], errors='coerce')\n",
    "\n",
    "average_metrics = ettm2_metrics.groupby('metric').agg({\n",
    "    'SOFTS': 'mean',\n",
    "    'TSMixer': 'mean',\n",
    "    'iTransformer': 'mean'\n",
    "}).reset_index()\n",
    "\n",
    "average_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAJOCAYAAACqS2TfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABZB0lEQVR4nO3de1hVVeL/8c8B5SIKXlC8IZKaeC3FSjC6TIJRmZolY4WamBqaEt/JJLW8pJhX1FHURkWbb4gTmjbDjGKTiYNdJOim01hZmB1CnMQLiQr794dfz6/TARNlewTfr+fZz8Nee+211zrVjg9rn70shmEYAgAAAAAA1c7F2R0AAAAAAKC2InQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGCSOs7uwPWovLxcP/zwgxo0aCCLxeLs7gAAAAAArjOGYejkyZNq2bKlXFwqn88mdFfghx9+kL+/v7O7AQAAAAC4zh0+fFitW7eu9DihuwINGjSQdOHD8/b2dnJvAAAAAADXmxMnTsjf39+WHytD6K7AxUfKvb29Cd0AAAAAgEr91leSeZEaAAAAAAAmcXroXrFihQIDA+Xh4aHg4GBlZWVVWnfPnj3q06ePmjRpIk9PTwUFBWnx4sV2dVJSUmSxWBy2M2fOmD0UAAAAAADsOPXx8rS0NMXFxWnFihXq06ePVq1apcjISO3fv19t2rRxqO/l5aXx48ere/fu8vLy0p49ezRmzBh5eXlp9OjRtnre3t768ssv7c718PAwfTwAAAAAAPySxTAMw1kXv+OOO9SzZ08lJyfbyjp16qSBAwcqMTHxstp45JFH5OXlpddff13ShZnuuLg4HT9+/Ir7deLECfn4+Ki4uJjvdAMAAACo8crKynTu3Dlnd6NGqVu3rlxdXSs9frm50Wkz3WfPnlVOTo4mT55sVx4REaHs7OzLaiM3N1fZ2dl65ZVX7MpPnTqlgIAAlZWV6dZbb9WsWbPUo0ePStspLS1VaWmpbf/EiRNVGAkAAAAAXJ8Mw1BBQcFVTUreyBo2bKjmzZv/5svSLsVpobuoqEhlZWXy8/OzK/fz81NBQcElz23durWOHj2q8+fPa/r06Ro1apTtWFBQkFJSUtStWzedOHFCS5YsUZ8+ffTJJ5+oQ4cOFbaXmJioGTNmXP2gAAAAAOA6cjFwN2vWTPXq1buq8HgjMQxDJSUlKiwslCS1aNHiitty+pJhv/6HbhjGb/6LkJWVpVOnTun999/X5MmT1b59ew0dOlSS1Lt3b/Xu3dtWt0+fPurZs6eWLVumpUuXVtheQkKC4uPjbfsX11sDAAAAgJqqrKzMFribNGni7O7UOJ6enpKkwsJCNWvW7JKPml+K00K3r6+vXF1dHWa1CwsLHWa/fy0wMFCS1K1bN/3444+aPn26LXT/mouLi2677TYdPHiw0vbc3d3l7u5exREAAAAAwPXr4ne469Wr5+Se1FwXP7tz585dceh22pJhbm5uCg4OVmZmpl15ZmamQkNDL7sdwzDsvo9d0fG8vLyrehwAAAAAAGoqHim/ctXx2Tn18fL4+HhFR0erV69eCgkJ0erVq5Wfn6+xY8dKuvDY95EjR7RhwwZJ0vLly9WmTRsFBQVJurBu94IFC/Tss8/a2pwxY4Z69+6tDh066MSJE1q6dKny8vK0fPnyaz9AAAAAAMANzamhOyoqSseOHdPMmTNltVrVtWtXZWRkKCAgQJJktVqVn59vq19eXq6EhAQdOnRIderUUbt27TR37lyNGTPGVuf48eMaPXq0CgoK5OPjox49emj37t26/fbbr/n4AAAAAOB6VFxcrJKSkmt2vXr16snHx+eaXe964tR1uq9XrNMNAAAA1BwrVqzQ/PnzZbVa1aVLFyUlJSksLKzCunv27NELL7ygf//73yopKVFAQIDGjBmj5557zlZn8+bNmjNnjr766iudO3dOHTp00P/8z/8oOjraVufkyZOaNm2atmzZosLCQvXo0UNLlizRbbfdZvp4L9eZM2d06NAhBQYGysPDw1ZeXFyspUuXq7z82q3b7eJSVxMmjKtxwbuyz1CqAet0AwAAAMDVSktLU1xcnFasWKE+ffpo1apVioyM1P79+9WmTRuH+l5eXho/fry6d+8uLy8v7dmzR2PGjJGXl5dGjx4tSWrcuLGmTJmioKAgubm56a9//aueeuopNWvWTP369ZMkjRo1Sp9//rlef/11tWzZUn/+85/Vt29f7d+/X61atbqmn0FVlZSUqLz8nNLTB6moqKnp1/P1ParBg7eopKSkxoXu6sBMdwWY6QYAAABqhjvuuEM9e/ZUcnKyraxTp04aOHCgEhMTL6uNRx55RF5eXnr99dcrrdOzZ089+OCDmjVrln7++Wc1aNBAW7du1YMPPmirc+utt+qhhx7SK6+8cuUDqkaVzdJarVatXr1aq1aNltVq/gunW7SwasyY1Ro9evRlv+D6nnvuUbdu3eTq6qr169fLzc1Ns2bN0hNPPKHx48frzTffVLNmzfTHP/5RkZGRKisr0+jRo/XPf/5TBQUFatOmjWJjYzVx4kS7dtetW6d58+bp0KFDatu2rSZMmKDY2NhK+1EdM91Oe3s5AAAAAFyNs2fPKicnRxEREXblERERys7Ovqw2cnNzlZ2drbvvvrvC44Zh6J133tGXX36pu+66S5J0/vx5lZWVOYQwT09P7dmz5wpGgoqsX79evr6++vDDD/Xss8/qmWee0WOPPabQ0FB9/PHH6tevn6Kjo/9v5r5crVu31qZNm7R//3699NJLevHFF7Vp0yZbe6+99pqmTJmi2bNn68CBA5ozZ46mTZum9evXmzoOHi8HAAAAUCMVFRWprKxMfn5+duV+fn4qKCi45LmtW7fW0aNHdf78eU2fPl2jRo2yO15cXKxWrVqptLRUrq6uWrFihcLDwyVJDRo0UEhIiGbNmqVOnTrJz89Pqamp+uCDD9ShQ4fqHeQN7JZbbtHUqVMlXVjZau7cufL19dXTTz8tSXrppZeUnJysTz/9VL1799aMGTNs5wYGBio7O1ubNm3SkCFDJEmzZs3SwoUL9cgjj9jq7N+/X6tWrdLw4cNNGwehGwAAAECN9uu1lA3D+M31lbOysnTq1Cm9//77mjx5stq3b6+hQ4fajjdo0EB5eXk6deqU3nnnHcXHx+umm27SPffcI0l6/fXXNXLkSLVq1Uqurq7q2bOnHn/8cX388cfVPr4bVffu3W0/u7q6qkmTJurWrZut7OIfWwoLCyVJK1eu1J/+9Cd99913+vnnn3X27FndeuutkqSjR4/q8OHDiomJsYV26cJTC2Z/z5zQDQAAAKBG8vX1laurq8OsdmFhocPs968FBgZKkrp166Yff/xR06dPtwvdLi4uat++vaQL39U+cOCAEhMTbaG7Xbt2eu+993T69GmdOHFCLVq0UFRUlK1dXL26deva7VssFruyi39YKS8v16ZNm/Tcc89p4cKFCgkJUYMGDTR//nx98MEHtjrShUfM77jjDrt2XV1dzRwGoRsAAABAzeTm5qbg4GBlZmZq0KBBtvLMzEwNGDDgstsxDEOlpaVXVMfLy0teXl766aeftH37ds2bN+/yB4Bqk5WVpdDQULuXon399de2n/38/NSqVSt98803euKJJ65p3wjdAAAAAGqs+Ph4RUdHq1evXgoJCdHq1auVn5+vsWPHSrrwXeAjR45ow4YNkqTly5erTZs2CgoKknRh3e4FCxbo2WeftbWZmJioXr16qV27djp79qwyMjK0YcMGuzekb9++XYZhqGPHjvrqq6/0/PPPq2PHjnrqqaeu4ehxUfv27bVhwwZt375dgYGBev311/XRRx/ZPXkwffp0TZgwQd7e3oqMjFRpaan27dunn376SfHx8ab1jdANAAAAoMaKiorSsWPHNHPmTFmtVnXt2lUZGRkKCAiQdGF5rPz8fFv98vJyJSQk6NChQ6pTp47atWunuXPnasyYMbY6p0+fVmxsrL7//nt5enoqKChIf/7znxUVFWWrU1xcrISEBH3//fdq3LixBg8erNmzZzs8En098/U9WmuuM3bsWOXl5SkqKkoWi0VDhw5VbGys/v73v9vqjBo1SvXq1dP8+fM1adIkeXl5qVu3boqLizO1b6zTXQHW6QYAAADMU1xcrJKSEmd347pSr169an+hV2VrTBcXF2vp0uUqLz9Xrde7FBeXupowYZzpLy2rbtWxTjcz3QAAAACuGWcEvprgWoZSHx8fTZgw7pr+4cOMPyrUFIRuAAAAANdMSUmJysvPKT19kIqKmjq7O9cFX9+jGjx4i0pKSq5ZMPXx8blhQ/C1RugGAAAAcM0VFTWV1drC2d0ATOfi7A4AAAAAAFBbEbphZ8WKFbaXBAQHBysrK6vSunv27FGfPn3UpEkT21sdFy9ebFfntddeU1hYmBo1aqRGjRqpb9+++vDDD+3qnD9/XlOnTlVgYKA8PT110003aebMmbYF7AEAAACgpuLxctikpaUpLi5OK1asUJ8+fbRq1SpFRkZq//79atOmjUN9Ly8vjR8/Xt27d5eXl5f27NmjMWPGyMvLS6NHj5Yk7dq1S0OHDlVoaKg8PDw0b948RURE6IsvvlCrVq0kSa+++qpWrlyp9evXq0uXLtq3b5+eeuop+fj4aOLEidf0MwAAAACA6kTohs2iRYsUExOjUaNGSZKSkpK0fft2JScnKzEx0aF+jx491KNHD9t+27ZttXnzZmVlZdlC9//+7//anfPaa6/pzTff1DvvvKNhw4ZJkvbu3asBAwbowQcftLWTmpqqffv2mTJOAAAAALhWeLwckqSzZ88qJydHERERduURERHKzs6+rDZyc3OVnZ2tu+++u9I6JSUlOnfunBo3bmwru/POO/XOO+/oP//5jyTpk08+0Z49e/TAAw9cwUgAAAAA4PrBTDckSUVFRSorK5Ofn59duZ+fnwoKCi55buvWrXX06FGdP39e06dPt82UV2Ty5Mlq1aqV+vbtayt74YUXVFxcrKCgILm6uqqsrEyzZ8/W0KFDr25QAAAAAOBkhG7YsVgsdvuGYTiU/VpWVpZOnTql999/X5MnT1b79u0rDMzz5s1Tamqqdu3aJQ8PD1t5Wlqa/vznP+uNN95Qly5dlJeXp7i4OLVs2VLDhw+vnoEBAAAAsCkuLlZJSck1u169evVu2HXBCd2QJPn6+srV1dVhVruwsNBh9vvXAgMDJUndunXTjz/+qOnTpzuE7gULFmjOnDnauXOnunfvbnfs+eef1+TJk/X73//e1s53332nxMREQjcAAABQzYqLi7X0j0tVfv7arRbkUsdFE8ZPuCGDN6EbkiQ3NzcFBwcrMzNTgwYNspVnZmZqwIABl92OYRgqLS21K5s/f75eeeUVbd++Xb169XI4p6SkRC4u9q8XcHV1ZckwAAAAwAQlJSUqP1+udKWrSEWmX89Xvhp8frBKSkouO3Tfc8896tatm1xdXbV+/Xq5ublp1qxZeuKJJzR+/Hi9+eabatasmf74xz8qMjJSP/30k8aPH68dO3bo1KlTat26tV588UU99dRTkqQjR44oPj5eO3bskIuLi+68804tWbJEbdu2NXHkFxC6YRMfH6/o6Gj16tVLISEhWr16tfLz8zV27FhJUkJCgo4cOaINGzZIkpYvX642bdooKChI0oV1uxcsWKBnn33W1ua8efM0bdo0vfHGG2rbtq1tJr1+/fqqX7++JKl///6aPXu22rRpoy5duig3N1eLFi3SyJEjr+XwAQAAgBtKkYpkldXZ3ajU+vXrNWnSJH344YdKS0vTM888o7feekuDBg3Siy++qMWLFys6Olr5+fmaNm2a9u/fr7///e/y9fXVV199pZ9//lnShT8y3HvvvQoLC9Pu3btVp04dvfLKK7r//vv16aefys3NzdRxELphExUVpWPHjmnmzJmyWq3q2rWrMjIyFBAQIEmyWq3Kz8+31S8vL1dCQoIOHTqkOnXqqF27dpo7d67GjBljq7NixQqdPXtWjz76qN21Xn75ZU2fPl2StGzZMk2bNk2xsbEqLCxUy5YtNWbMGL300kvmDxoAAADAdemWW27R1KlTJV2YAJw7d658fX319NNPS5JeeuklJScn69NPP1V+fr569Ohhe7L2lzPYGzdulIuLi/70pz/Z3le1bt06NWzYULt27XJYwam6EbphJzY2VrGxsRUeS0lJsdt/9tln7Wa1K/Ltt9/+5jUbNGigpKQkJSUlXWYvAQAAANR2v3wXlKurq5o0aaJu3brZyi6+e6qwsFDPPPOMBg8erI8//lgREREaOHCgQkNDJUk5OTn66quv1KBBA7v2z5w5o6+//tr0cRC6a7hr/dbBmuBGfjMiAAAAUFvUrVvXbt9isdiVXZy1Li8vV2RkpL777jv97W9/086dO3Xfffdp3LhxWrBggcrLyxUcHKz//d//dbhG06ZNzR2ECN01WnFxsZYuXa7y8nPO7sp1xcWlriZMGEfwBgAAAG4gTZs21YgRIzRixAiFhYXp+eef14IFC9SzZ0+lpaWpWbNm8vb2vub9InTXYCUlJSovP6f09EEqKjL/LzQ1ga/vUQ0evKVKb0YEAAAAULO99NJLCg4OVpcuXVRaWqq//vWv6tSpkyTpiSee0Pz58zVgwADNnDlTrVu3Vn5+vjZv3qznn39erVu3NrVvhO5aoKioqazWFs7uBgAAAIAaxFe+teY6bm5uSkhI0LfffitPT0+FhYVp48aNki58/XT37t164YUX9Mgjj+jkyZNq1aqV7rvvvmsy803oBgAAAIAbSL169eRSx0WDzw++Ztd0qeOievXqXXb9Xbt2OZRV9JJmwzAkSQMHDrS96bwizZs31/r16y/7+tWJ0A0AAAAANxAfHx9NGD/hmr6Q+UZ+2TGhGwAAAABuMD4+PjdsCL7WXJzdAQAAAAAAaitCNwAAAAAAJiF0AwAAAABgEkI3AAAAANRiF9/wjaqrjs+O0A0AAAAAtVDdunUl6Zq+pby2ufjZXfwsrwRvLwcAAACAWsjV1VUNGzZUYWGhpAvLdlksFif3qmYwDEMlJSUqLCxUw4YN5erqesVtEboBAAAAoJZq3ry5JNmCN6qmYcOGts/wShG6AQAAAKCWslgsatGihZo1a6Zz5845uzs1St26da9qhvsiQjcAAAAA1HKurq7VEiBRdbxIDQAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADCJ00P3ihUrFBgYKA8PDwUHBysrK6vSunv27FGfPn3UpEkTeXp6KigoSIsXL3aol56ers6dO8vd3V2dO3fWli1bzBwCAAAAAAAVcmroTktLU1xcnKZMmaLc3FyFhYUpMjJS+fn5Fdb38vLS+PHjtXv3bh04cEBTp07V1KlTtXr1aludvXv3KioqStHR0frkk08UHR2tIUOG6IMPPrhWwwIAAAAAQJKTQ/eiRYsUExOjUaNGqVOnTkpKSpK/v7+Sk5MrrN+jRw8NHTpUXbp0Udu2bfXkk0+qX79+drPjSUlJCg8PV0JCgoKCgpSQkKD77rtPSUlJ12hUAAAAAABc4LTQffbsWeXk5CgiIsKuPCIiQtnZ2ZfVRm5urrKzs3X33Xfbyvbu3evQZr9+/S7ZZmlpqU6cOGG3AQAAAABwtZwWuouKilRWViY/Pz+7cj8/PxUUFFzy3NatW8vd3V29evXSuHHjNGrUKNuxgoKCKreZmJgoHx8f2+bv738FIwIAAAAAwJ7TX6RmsVjs9g3DcCj7taysLO3bt08rV65UUlKSUlNTr6rNhIQEFRcX27bDhw9XcRQAAAAAADiq46wL+/r6ytXV1WEGurCw0GGm+tcCAwMlSd26ddOPP/6o6dOna+jQoZKk5s2bV7lNd3d3ubu7X8kwAAAAAAColNNmut3c3BQcHKzMzEy78szMTIWGhl52O4ZhqLS01LYfEhLi0OaOHTuq1CYAAAAAANXBaTPdkhQfH6/o6Gj16tVLISEhWr16tfLz8zV27FhJFx77PnLkiDZs2CBJWr58udq0aaOgoCBJF9btXrBggZ599llbmxMnTtRdd92lV199VQMGDNDWrVu1c+dO7dmz59oPEAAAAABwQ3Nq6I6KitKxY8c0c+ZMWa1Wde3aVRkZGQoICJAkWa1WuzW7y8vLlZCQoEOHDqlOnTpq166d5s6dqzFjxtjqhIaGauPGjZo6daqmTZumdu3aKS0tTXfcccc1Hx8AAAAA4Mbm1NAtSbGxsYqNja3wWEpKit3+s88+azerXZlHH31Ujz76aHV0DwAAAACAK+b0t5cDAAAAAFBbEboBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAapkVK1YoMDBQHh4eCg4OVlZWVqV1N2/erPDwcDVt2lTe3t4KCQnR9u3b7ercc889slgsDtuDDz5o9lBqPEI3AAAAANQiaWlpiouL05QpU5Sbm6uwsDBFRkYqPz+/wvq7d+9WeHi4MjIylJOTo3vvvVf9+/dXbm6urc7mzZtltVpt2+effy5XV1c99thj12pYNVYdZ3cAAAAAAFB9Fi1apJiYGI0aNUqSlJSUpO3btys5OVmJiYkO9ZOSkuz258yZo61bt+rtt99Wjx49JEmNGze2q7Nx40bVq1eP0H0ZmOkGAAAAgFri7NmzysnJUUREhF15RESEsrOzL6uN8vJynTx50iFo/9KaNWv0+9//Xl5eXlfV3xsBM90AAAAAUEsUFRWprKxMfn5+duV+fn4qKCi4rDYWLlyo06dPa8iQIRUe//DDD/X5559rzZo1V93fGwGhGwAAAABqGYvFYrdvGIZDWUVSU1M1ffp0bd26Vc2aNauwzpo1a9S1a1fdfvvt1dLX2o7HywEAAACglvD19ZWrq6vDrHZhYaHD7PevpaWlKSYmRps2bVLfvn0rrFNSUqKNGzfavi+O30boBgAAAIBaws3NTcHBwcrMzLQrz8zMVGhoaKXnpaamasSIEXrjjTcuuQzYpk2bVFpaqieffLLa+lzb8Xg5AAAAANQi8fHxio6OVq9evRQSEqLVq1crPz9fY8eOlSQlJCToyJEj2rBhg6QLgXvYsGFasmSJevfubZsl9/T0lI+Pj13ba9as0cCBA9WkSZNrO6gajNANAAAAALVIVFSUjh07ppkzZ8pqtapr167KyMhQQECAJMlqtdqt2b1q1SqdP39e48aN07hx42zlw4cPV0pKim3/P//5j/bs2aMdO3Zcs7HUBoRuAAAAALgOFBUVVVtbgwYN0qBBg+zKrFarJNnW6r64n5qaWmk7F+tIUoMGDfTDDz84lJulXr16DjPtNRGhGwAAAACcqH79UypXuTZv3uzsrlxXXOq4aML4CTU+eBO6AQAAAMCJPDzOyEUuSle6ilR9s901ma98Nfj8YJWUlBC6AQAAAABXr0hFssr8x7ZxbbFkGAAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAO/YcWKFQoMDJSHh4eCg4OVlZVVad3NmzcrPDxcTZs2lbe3t0JCQrR9+3a7OikpKbJYLA7bmTNnbHWSk5PVvXt3eXt729r5+9//btoYAQAAAJiD0A1cQlpamuLi4jRlyhTl5uYqLCxMkZGRys/Pr7D+7t27FR4eroyMDOXk5Ojee+9V//79lZuba1fP29tbVqvVbvPw8LAdb926tebOnat9+/Zp3759+t3vfqcBAwboiy++MHW8AAAAAKpXHWd3ALieLVq0SDExMRo1apQkKSkpSdu3b1dycrISExMd6iclJdntz5kzR1u3btXbb7+tHj162MotFouaN29e6XX79+9vtz979mwlJyfr/fffV5cuXa5iRAAAAACuJWa6gUqcPXtWOTk5ioiIsCuPiIhQdnb2ZbVRXl6ukydPqnHjxnblp06dUkBAgFq3bq2HHnrIYSb8l8rKyrRx40adPn1aISEhVR8IAAAAAKdhphuoRFFRkcrKyuTn52dX7ufnp4KCgstqY+HChTp9+rSGDBliKwsKClJKSoq6deumEydOaMmSJerTp48++eQTdejQwVbvs88+U0hIiM6cOaP69etry5Yt6ty5c/UMDgAAAMA1QegGfoPFYrHbNwzDoawiqampmj59urZu3apmzZrZynv37q3evXvb9vv06aOePXtq2bJlWrp0qa28Y8eOysvL0/Hjx5Wenq7hw4frvffeI3gDAAAANQihG6iEr6+vXF1dHWa1CwsLHWa/fy0tLU0xMTH6y1/+or59+16yrouLi2677TYdPHjQrtzNzU3t27eXJPXq1UsfffSRlixZolWrVl3BaAAAAAA4A9/pBirh5uam4OBgZWZm2pVnZmYqNDS00vNSU1M1YsQIvfHGG3rwwQd/8zqGYSgvL08tWrT4zXqlpaWX13kA153qXn7wlzZu3CiLxaKBAwfalZ88eVJxcXEKCAiQp6enQkND9dFHH1XXkAAAwGUgdAOXEB8frz/96U9au3atDhw4oOeee075+fkaO3asJCkhIUHDhg2z1U9NTdWwYcO0cOFC9e7dWwUFBSooKFBxcbGtzowZM7R9+3Z98803ysvLU0xMjPLy8mxtStKLL76orKwsffvtt/rss880ZcoU7dq1S0888cS1GzyAamPW8oOS9N133+kPf/iDwsLCHI6NGjVKmZmZev311/XZZ58pIiJCffv21ZEjR6p9jAAAoGI8Xg5cQlRUlI4dO6aZM2fKarWqa9euysjIUEBAgCTJarXa/dK8atUqnT9/XuPGjdO4ceNs5cOHD1dKSook6fjx4xo9erQKCgrk4+OjHj16aPfu3br99ttt9X/88UdFR0fLarXKx8dH3bt31z/+8Q+Fh4dfm4EDqFZmLT9YVlamJ554QjNmzFBWVpaOHz9uO/bzzz8rPT1dW7du1V133SVJmj59ut566y0lJyfrlVdeqf6BAgAAB4Ru1EpFRUXV1tagQYM0aNAguzKr1SpJtl+WL+6npqZW2s7FOpMmTdKkSZMqPS5Jr7zySoW/EP+yzuWqV6+efHx8qnwegOpxcfnByZMn25VXx/KDM2fOVNOmTRUTE+PwuPr58+dVVlYmDw8Pu3JPT0/t2bPnCkYCAACuBKEbtUr9+qdUrnJt3rzZ2V25brjUcdGE8RMI3oCTmLX84L/+9S+tWbNGeXl5FZ7ToEEDhYSEaNasWerUqZP8/PyUmpqqDz74wG55QgAAYC6nh+4VK1Zo/vz5slqt6tKli5KSkir8Xpp04cUyycnJysvLU2lpqbp06aLp06erX79+tjopKSl66qmnHM79+eefHf7aj9rHw+OMXOSidKWrSNU3211T+cpXg88PVklJCaEbcLLqXH7w5MmTevLJJ/Xaa6/J19e30nNff/11jRw5Uq1atZKrq6t69uypxx9/XB9//PHVDQYAAFw2p4buiy+WWbFihfr06aNVq1YpMjJS+/fvV5s2bRzqX3yxzJw5c9SwYUOtW7dO/fv31wcffGD3HTdvb299+eWXducSuG8sRSqSVVV/FBsAqpsZyw9+/fXX+vbbb9W/f39bWXl5uSSpTp06+vLLL9WuXTu1a9dO7733nk6fPq0TJ06oRYsWioqKUmBgYDWOEAAAXIpTQ7dZL5axWCxq3ry5qX0HAOBy/HL5wV++HyIzM1MDBgyo9LzU1FSNHDlSqampDssPBgUF6bPPPrMrmzp1qk6ePKklS5bI39/f7piXl5e8vLz0008/afv27Zo3b141jAwAAFwOp4VuM18sc+rUKQUEBKisrEy33nqrZs2aZRfKf620tNRu/eMTJ05UYSQAAFxafHy8oqOj1atXL4WEhGj16tUOyw8eOXJEGzZskPT/lx9csmSJbflB6cJL0Hx8fOTh4aGuXbvaXaNhw4aSZFe+fft2GYahjh076quvvtLzzz+vjh07Vvg1LAAAYA6nrdNt1otlgoKClJKSom3btik1NVUeHh7q06ePDh48WGk7iYmJ8vHxsW2/niEAAOBqREVFKSkpSTNnztStt96q3bt3X/bygy1atLBtEydOrNJ1i4uLNW7cOAUFBWnYsGG68847tWPHDtWtW7daxwcAACrn9BepVeeLZSSpd+/e6t27t22/T58+6tmzp5YtW6alS5dW2FZCQoLi4+Nt+ydOnCB4A8ANrri4WCUlJdXWXnUvP/hrv25DksLCwhyWByspKbnicbEEIQAAVee00G3Gi2Uq4uLiottuu+2SM93u7u5yd3e//M4DAGq14uJiLV26XOXl55zdleuKi0tdTZgwjuANAEAVOC10m/FimYoYhqG8vDx169atWvoNAKj9SkpKVF5+Tunpg1RU1NTZ3bku+Poe1eDBW1iCEACAKnLq4+XV/WIZSZoxY4Z69+6tDh066MSJE1q6dKny8vK0fPly5wwSAFBjFRU1ldXawtndAAAANZhTQ3dUVJSOHTummTNnymq1qmvXrpf9Yplx48bZyocPH66UlBRJ0vHjxzV69GgVFBTIx8dHPXr00O7du3X77bdf07EBAAAAAOD0F6nFxsYqNja2wmMXg/RFu3bt+s32Fi9erMWLF1dDzwAAAAAAuDpOWzIMAAAAAIDajtANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJiE0A0AAAAAgEkI3QAAAAAAmITQDQAAAACASQjdAAAAAACYhNANAAAAAIBJCN0AAAAAAJjE6aF7xYoVCgwMlIeHh4KDg5WVlVVp3c2bNys8PFxNmzaVt7e3QkJCtH37dod66enp6ty5s9zd3dW5c2dt2bLFzCEAAAAAAFAhp4butLQ0xcXFacqUKcrNzVVYWJgiIyOVn59fYf3du3crPDxcGRkZysnJ0b333qv+/fsrNzfXVmfv3r2KiopSdHS0PvnkE0VHR2vIkCH64IMPrtWwAAAAAACQ5OTQvWjRIsXExGjUqFHq1KmTkpKS5O/vr+Tk5ArrJyUladKkSbrtttvUoUMHzZkzRx06dNDbb79tVyc8PFwJCQkKCgpSQkKC7rvvPiUlJV2jUQEAAAAAcIHTQvfZs2eVk5OjiIgIu/KIiAhlZ2dfVhvl5eU6efKkGjdubCvbu3evQ5v9+vW7ZJulpaU6ceKE3QYAAAAAwNVyWuguKipSWVmZ/Pz87Mr9/PxUUFBwWW0sXLhQp0+f1pAhQ2xlBQUFVW4zMTFRPj4+ts3f378KIwEAAAAAoGJOf5GaxWKx2zcMw6GsIqmpqZo+fbrS0tLUrFmzq2ozISFBxcXFtu3w4cNVGAEAAAAAABWr46wL+/r6ytXV1WEGurCw0GGm+tfS0tIUExOjv/zlL+rbt6/dsebNm1e5TXd3d7m7u1dxBAAAAAAAXJrTZrrd3NwUHByszMxMu/LMzEyFhoZWel5qaqpGjBihN954Qw8++KDD8ZCQEIc2d+zYcck2AQAAAAAwg9NmuiUpPj5e0dHR6tWrl0JCQrR69Wrl5+dr7Nixki489n3kyBFt2LBB0oXAPWzYMC1ZskS9e/e2zWh7enrKx8dHkjRx4kTdddddevXVVzVgwABt3bpVO3fu1J49e5wzSAAAAADADcup3+mOiopSUlKSZs6cqVtvvVW7d+9WRkaGAgICJElWq9Vuze5Vq1bp/PnzGjdunFq0aGHbJk6caKsTGhqqjRs3at26derevbtSUlKUlpamO+6445qPDwAAAABwY3PqTLckxcbGKjY2tsJjKSkpdvu7du26rDYfffRRPfroo1fZMwAAAAAAro7T314OAAAAAEBtRegGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJFUO3YcPH9b3339v2//www8VFxen1atXV2vHAAAAAACo6aocuh9//HG9++67kqSCggKFh4frww8/1IsvvqiZM2dWewcBAAAAAKipqhy6P//8c91+++2SpE2bNqlr167Kzs7WG2+8oZSUlOruHwAAAAAANVaVQ/e5c+fk7u4uSdq5c6cefvhhSVJQUJCsVmv19g4AAAAAgBqsyqG7S5cuWrlypbKyspSZman7779fkvTDDz+oSZMm1d5BAAAAAABqqiqH7ldffVWrVq3SPffco6FDh+qWW26RJG3bts322DkAAAAAAJDqVPWEe+65R0VFRTpx4oQaNWpkKx89erTq1atXrZ0DAAAAAKAmu6J1ug3DUE5OjlatWqWTJ09Kktzc3AjdAAAAAAD8QpVnur/77jvdf//9ys/PV2lpqcLDw9WgQQPNmzdPZ86c0cqVK83oJwAAAAAANU6VZ7onTpyoXr166aeffpKnp6etfNCgQXrnnXeqtXMAAAAAANRkVZ7p3rNnj/71r3/Jzc3NrjwgIEBHjhypto4BAAAAAFDTVXmmu7y8XGVlZQ7l33//vRo0aFAtnQIAAAAAoDaocugODw9XUlKSbd9isejUqVN6+eWX9cADD1Rn3wAAAAAAqNGq/Hj54sWLde+996pz5846c+aMHn/8cR08eFC+vr5KTU01o48AAAAAANRIVQ7dLVu2VF5enlJTU/Xxxx+rvLxcMTExeuKJJ+xerAYAAAAAwI2uyqFbkjw9PTVy5EiNHDmyuvsDAAAAAECtUeXQvWHDhkseHzZs2BV3BgAAAACA2qTKoXvixIl2++fOnVNJSYnc3NxUr149QjcAAAAAAP+nym8v/+mnn+y2U6dO6csvv9Sdd97Ji9QAAAAAAPiFKofuinTo0EFz5851mAUHAAAAAOBGVi2hW5JcXV31ww8/VFdzAAAAAADUeFX+Tve2bdvs9g3DkNVq1R//+Ef16dOn2joGAAAAAEBNV+XQPXDgQLt9i8Wipk2b6ne/+50WLlxYXf0CAAAAAKDGq3LoLi8vN6MfAAAAAADUOtX2nW4AAAAAAGDvsma64+PjL7vBRYsWXXFnAAAAAACoTS5rpjs3N/eytry8PJO7CwAAaosVK1YoMDBQHh4eCg4OVlZWVqV1rVarHn/8cXXs2FEuLi6Ki4ursN7x48c1btw4tWjRQh4eHurUqZMyMjJsx9u2bSuLxeKwjRs3rrqHBwCApMuc6X733XfN7gcAALiBpKWlKS4uTitWrFCfPn20atUqRUZGav/+/WrTpo1D/dLSUjVt2lRTpkzR4sWLK2zz7NmzCg8PV7NmzfTmm2+qdevWOnz4sBo0aGCr89FHH6msrMy2//nnnys8PFyPPfZY9Q8SAABdwYvUAAAArtaiRYsUExOjUaNGSZKSkpK0fft2JScnKzEx0aF+27ZttWTJEknS2rVrK2xz7dq1+u9//6vs7GzVrVtXkhQQEGBXp2nTpnb7c+fOVbt27XT33Xdf9ZgAAKjIFYXujz76SH/5y1+Un5+vs2fP2h3bvHlztXQMAADUTmfPnlVOTo4mT55sVx4REaHs7Owrbnfbtm0KCQnRuHHjtHXrVjVt2lSPP/64XnjhBbm6ulbYjz//+c+Kj4+XxWK54usCAHApVX57+caNG9WnTx/t379fW7Zs0blz57R//37985//lI+Pjxl9BAAAtUhRUZHKysrk5+dnV+7n56eCgoIrbvebb77Rm2++qbKyMmVkZGjq1KlauHChZs+eXWH9t956S8ePH9eIESOu+JoAAPyWKs90z5kzR4sXL9a4cePUoEEDLVmyRIGBgRozZoxatGhhRh8BAEAt9OvZZcMwrmrGuby8XM2aNdPq1avl6uqq4OBg/fDDD5o/f75eeuklh/pr1qxRZGSkWrZsecXXBADgt1R5pvvrr7/Wgw8+KElyd3fX6dOnZbFY9Nxzz2n16tXV3kEAAFC7+Pr6ytXV1WFWu7Cw0GH2uypatGihm2++2e5R8k6dOqmgoMDh63Dfffeddu7caftOOQAAZqly6G7cuLFOnjwpSWrVqpU+//xzSReW6CgpKane3gEAgFrHzc1NwcHByszMtCvPzMxUaGjoFbfbp08fffXVVyovL7eV/ec//1GLFi3k5uZmV3fdunVq1qyZbSIBAACzXHbovrgGd1hYmO1/kkOGDNHEiRP19NNPa+jQobrvvvtM6SQAAKhd4uPj9ac//Ulr167VgQMH9Nxzzyk/P19jx46VJCUkJGjYsGF25+Tl5SkvL0+nTp3S0aNHlZeXp/3799uOP/PMMzp27JgmTpyo//znP/rb3/6mOXPmOKzBXV5ernXr1mn48OGqU4eFXAAA5rrs/9P07NlTPXr00MCBAzV06FBJF/6HWLduXe3Zs0ePPPKIpk2bZlpHAQBA7REVFaVjx45p5syZslqt6tq1qzIyMmxLfFmtVuXn59ud06NHD9vPOTk5euONNxQQEKBvv/1WkuTv768dO3boueeeU/fu3dWqVStNnDhRL7zwgl07O3fuVH5+vkaOHGnuIAEAUBVC97/+9S+tXbtWCxYsUGJioh555BHFxMRo0qRJmjRpkpl9BAAA14mioqJqa2vQoEEaNGiQXZnVapUk21rdF/cl6YcffqiwnV/Wadu2rbZs2WJ3vLCw0G6/W7dutrZ+ee6VqFevHqu3AAAu6bJDd0hIiEJCQrR06VJt2rRJ69atU9++fdW2bVuNHDlSw4cPV+vWrc3sKwAAcJL69U+pXOXavHmzs7tyXXGp46IJ4ycQvAEAlaryF5k8PT01fPhwDR8+XF9//bXWrVunVatWafr06QoPD1dGRoYZ/QQAAE7k4XFGLnJRutJVpOqb7a7JfOWrwecHq6SkhNANAKjUVb09pF27dpo8ebL8/f314osvavv27dXVLwAAcB0qUpGsurpHsgEAuJFcceh+7733tHbtWqWnp8vV1VVDhgxRTExMdfYNAAAAAIAarUqh+/Dhw0pJSVFKSooOHTqk0NBQLVu2TEOGDJGXl5dZfQQAAAAAoEa67NAdHh6ud999V02bNtWwYcM0cuRIdezY0cy+AQAAAABQo1126Pb09FR6eroeeughubq6mtknAAAAAABqhcsO3du2bTOzHwAAAAAA1Douzu4AAAAAAAC1FaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkzg9dK9YsUKBgYHy8PBQcHCwsrKyKq1rtVr1+OOPq2PHjnJxcVFcXJxDnZSUFFksFoftzJkzJo4CAAAAAABHTg3daWlpiouL05QpU5Sbm6uwsDBFRkYqPz+/wvqlpaVq2rSppkyZoltuuaXSdr29vWW1Wu02Dw8Ps4YBAAAAAECFnBq6Fy1apJiYGI0aNUqdOnVSUlKS/P39lZycXGH9tm3basmSJRo2bJh8fHwqbddisah58+Z2GwAAAAAA15rTQvfZs2eVk5OjiIgIu/KIiAhlZ2dfVdunTp1SQECAWrdurYceeki5ublX1R4AAAAAAFfCaaG7qKhIZWVl8vPzsyv38/NTQUHBFbcbFBSklJQUbdu2TampqfLw8FCfPn108ODBSs8pLS3ViRMn7DYAAAAAAK6W01+kZrFY7PYNw3Aoq4revXvrySef1C233KKwsDBt2rRJN998s5YtW1bpOYmJifLx8bFt/v7+V3x9AAAAAAAuclro9vX1laurq8OsdmFhocPs99VwcXHRbbfddsmZ7oSEBBUXF9u2w4cPV9v1AQAAAAA3LqeFbjc3NwUHByszM9OuPDMzU6GhodV2HcMwlJeXpxYtWlRax93dXd7e3nYbAAAAAABXq44zLx4fH6/o6Gj16tVLISEhWr16tfLz8zV27FhJF2agjxw5og0bNtjOycvLk3ThZWlHjx5VXl6e3Nzc1LlzZ0nSjBkz1Lt3b3Xo0EEnTpzQ0qVLlZeXp+XLl1/z8QEAAAAAbmxODd1RUVE6duyYZs6cKavVqq5duyojI0MBAQGSJKvV6rBmd48ePWw/5+Tk6I033lBAQIC+/fZbSdLx48c1evRoFRQUyMfHRz169NDu3bt1++23X7NxAQAAAAAgOTl0S1JsbKxiY2MrPJaSkuJQZhjGJdtbvHixFi9eXB1dAwAAAADgqjj97eUAAAAAANRWhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCSEbgAAAAAATELoBgAAAADAJIRuAAAAAABMQugGAAAAAMAkhG4AAAAAAExC6AYAAAAAwCROD90rVqxQYGCgPDw8FBwcrKysrErrWq1WPf744+rYsaNcXFwUFxdXYb309HR17txZ7u7u6ty5s7Zs2WJS7wEAAAAAqJxTQ3daWpri4uI0ZcoU5ebmKiwsTJGRkcrPz6+wfmlpqZo2baopU6bolltuqbDO3r17FRUVpejoaH3yySeKjo7WkCFD9MEHH5g5FAAAAAAAHDg1dC9atEgxMTEaNWqUOnXqpKSkJPn7+ys5ObnC+m3bttWSJUs0bNgw+fj4VFgnKSlJ4eHhSkhIUFBQkBISEnTfffcpKSnJxJEAAAAAAODIaaH77NmzysnJUUREhF15RESEsrOzr7jdvXv3OrTZr1+/q2oTAAAAAIArUcdZFy4qKlJZWZn8/Pzsyv38/FRQUHDF7RYUFFS5zdLSUpWWltr2T5w4ccXXBwAAAADgIqe/SM1isdjtG4bhUGZ2m4mJifLx8bFt/v7+V3V9AAAAAAAkJ4ZuX19fubq6OsxAFxYWOsxUV0Xz5s2r3GZCQoKKi4tt2+HDh6/4+gAAAAAAXOS00O3m5qbg4GBlZmbalWdmZio0NPSK2w0JCXFoc8eOHZds093dXd7e3nYbAAAAAABXy2nf6Zak+Ph4RUdHq1evXgoJCdHq1auVn5+vsWPHSrowA33kyBFt2LDBdk5eXp4k6dSpUzp69Kjy8vLk5uamzp07S5ImTpyou+66S6+++qoGDBigrVu3aufOndqzZ881Hx8AAAAA4Mbm1NAdFRWlY8eOaebMmbJareratasyMjIUEBAgSbJarQ5rdvfo0cP2c05Ojt544w0FBATo22+/lSSFhoZq48aNmjp1qqZNm6Z27dopLS1Nd9xxxzUbFwAAAAAAkpNDtyTFxsYqNja2wmMpKSkOZYZh/Gabjz76qB599NGr7RoAAAAAAFfF6W8vBwAAAACgtiJ0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmIXQDAAAAAGASQjcAAAAAACYhdAMAAAAAYBJCNwAAAAAAJiF0AwAAAABgEkI3AAAAAAAmcXroXrFihQIDA+Xh4aHg4GBlZWVdsv57772n4OBgeXh46KabbtLKlSvtjqekpMhisThsZ86cMXMYAAAAAAA4cGroTktLU1xcnKZMmaLc3FyFhYUpMjJS+fn5FdY/dOiQHnjgAYWFhSk3N1cvvviiJkyYoPT0dLt63t7eslqtdpuHh8e1GBIAAAAAADZ1nHnxRYsWKSYmRqNGjZIkJSUlafv27UpOTlZiYqJD/ZUrV6pNmzZKSkqSJHXq1En79u3TggULNHjwYFs9i8Wi5s2bX5MxAAAAAABQGafNdJ89e1Y5OTmKiIiwK4+IiFB2dnaF5+zdu9ehfr9+/bRv3z6dO3fOVnbq1CkFBASodevWeuihh5Sbm1v9AwAAAAAA4Dc4LXQXFRWprKxMfn5+duV+fn4qKCio8JyCgoIK658/f15FRUWSpKCgIKWkpGjbtm1KTU2Vh4eH+vTpo4MHD1bal9LSUp04ccJuAwAAAADgajn9RWoWi8Vu3zAMh7Lfqv/L8t69e+vJJ5/ULbfcorCwMG3atEk333yzli1bVmmbiYmJ8vHxsW3+/v5XOhwAAAAAAGycFrp9fX3l6urqMKtdWFjoMJt9UfPmzSusX6dOHTVp0qTCc1xcXHTbbbddcqY7ISFBxcXFtu3w4cNVHA0AAAAAAI6cFrrd3NwUHByszMxMu/LMzEyFhoZWeE5ISIhD/R07dqhXr16qW7duhecYhqG8vDy1aNGi0r64u7vL29vbbgMAAAAA4Go59fHy+Ph4/elPf9LatWt14MABPffcc8rPz9fYsWMlXZiBHjZsmK3+2LFj9d133yk+Pl4HDhzQ2rVrtWbNGv3hD3+w1ZkxY4a2b9+ub775Rnl5eYqJiVFeXp6tTQAAAAAArhWnLhkWFRWlY8eOaebMmbJareratasyMjIUEBAgSbJarXZrdgcGBiojI0PPPfecli9frpYtW2rp0qV2y4UdP35co0ePVkFBgXx8fNSjRw/t3r1bt99++zUfHwAAAADgxubU0C1JsbGxio2NrfBYSkqKQ9ndd9+tjz/+uNL2Fi9erMWLF1dX9wAAAAAAuGJOf3s5AAAAAAC1FaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADCJ00P3ihUrFBgYKA8PDwUHBysrK+uS9d977z0FBwfLw8NDN910k1auXOlQJz09XZ07d5a7u7s6d+6sLVu2mNV9AAAAAAAq5dTQnZaWpri4OE2ZMkW5ubkKCwtTZGSk8vPzK6x/6NAhPfDAAwoLC1Nubq5efPFFTZgwQenp6bY6e/fuVVRUlKKjo/XJJ58oOjpaQ4YM0QcffHCthgUAAAAAgCQnh+5FixYpJiZGo0aNUqdOnZSUlCR/f38lJydXWH/lypVq06aNkpKS1KlTJ40aNUojR47UggULbHWSkpIUHh6uhIQEBQUFKSEhQffdd5+SkpKu0agAAAAAALjAaaH77NmzysnJUUREhF15RESEsrOzKzxn7969DvX79eunffv26dy5c5esU1mbAAAAAACYpY6zLlxUVKSysjL5+fnZlfv5+amgoKDCcwoKCiqsf/78eRUVFalFixaV1qmsTUkqLS1VaWmpbb+4uFiSdOLEiSqN6Vo7efKkzpw5o0aNDqm8/KSzu3NdqF//+wufiRqpXOXO7o7TNVIjndEZnTx5Ul5eXs7uDlBjcH91xP3VEfdY4Mpwj3XEPdZRTbjHXsyLhmFcsp7TQvdFFovFbt8wDIey36r/6/KqtpmYmKgZM2Y4lPv7+1fe8evKXGd34Lrx44/S3r3O7sX140f9qP3ar7lz+XcEuDL8t3MR91dH3GOBq8V/Oxdxj3VUk+6xJ0+elI+PT6XHnRa6fX195erq6jADXVhY6DBTfVHz5s0rrF+nTh01adLkknUqa1OSEhISFB8fb9svLy/Xf//7XzVp0uSSYR3XnxMnTsjf31+HDx+Wt7e3s7sDALUG91cAMA/32JrJMAydPHlSLVu2vGQ9p4VuNzc3BQcHKzMzU4MGDbKVZ2ZmasCAARWeExISorffftuubMeOHerVq5fq1q1rq5OZmannnnvOrk5oaGilfXF3d5e7u7tdWcOGDas6JFxHvL29uWEBgAm4vwKAebjH1jyXmuG+yKmPl8fHxys6Olq9evVSSEiIVq9erfz8fI0dO1bShRnoI0eOaMOGDZKksWPH6o9//KPi4+P19NNPa+/evVqzZo1SU1NtbU6cOFF33XWXXn31VQ0YMEBbt27Vzp07tWfPHqeMEQAAAABw43Jq6I6KitKxY8c0c+ZMWa1Wde3aVRkZGQoICJAkWa1WuzW7AwMDlZGRoeeee07Lly9Xy5YttXTpUg0ePNhWJzQ0VBs3btTUqVM1bdo0tWvXTmlpabrjjjuu+fgAAAAAADc2i/Fbr1oDapDS0lIlJiYqISHB4SsDAIArx/0VAMzDPbZ2I3QDAAAAAGASF2d3AAAAAACA2orQDQAAAACASQjdAACgSlJSUlhaE0CNcM899yguLs7Z3ahUQUGBwsPD5eXlxX21FiN0w6kKCws1ZswYtWnTRu7u7mrevLn69eunvXv32upkZ2frgQceUKNGjeTh4aFu3bpp4cKFKisrs2vLYrE4bHfeeadGjBhR4bFfbpfbFwC4nvzWvW3EiBGSpHfffVf33nuvGjdurHr16qlDhw4aPny4zp8/L0natWuXLBaLGjVqpDNnzthd48MPP7S7V0oXVh/5z3/+c83GCQBXavPmzYqJifnN++X06dOd0r/FixfLarUqLy+P+2ot5tQlw4DBgwfr3LlzWr9+vW666Sb9+OOPeuedd/Tf//5XkrRlyxYNGTJETz31lN599101bNhQO3fu1KRJk/T+++9r06ZNdr8Irlu3Tvfff79t383NTa6urpo7d66trEWLFg71LqcvAHC9sVqttp/T0tL00ksv6csvv7SVeXp66osvvlBkZKQmTJigZcuWydPTUwcPHtSbb76p8vJyu/YaNGigLVu2aOjQobaytWvXqk2bNnZLeHp6esrT09PEkUnnzp1T3bp1Tb0GgNqvcePG8vHxsbtfLliwQP/4xz+0c+dOW1n9+vVtPxuGobKyMtWpY35U+vrrrxUcHKwOHTpccRvX+n7J/fkKGICT/PTTT4YkY9euXRUeP3XqlNGkSRPjkUcecTi2bds2Q5KxceNGW5kkY8uWLb953Yrq/VZfAOB6t27dOsPHx8ehfPHixUbbtm0vee67775rSDKmTp1q9O3b11ZeUlJi+Pj4GNOmTTN++SvDL69VXl5u3HfffUa/fv2M8vJywzAu3FP9/f2NF1980XbO2rVrjaCgIMPd3d3o2LGjsXz5ctuxQ4cOGZKMtLQ04+677zbc3d2NtWvXXsnHAAB27r77bmPixIl2ZS+//LJxyy232PYv3gP/8Y9/GMHBwUbdunWNf/7zn8ZXX31lPPzww0azZs0MLy8vo1evXkZmZqZdWwEBAcbs2bONp556yqhfv77h7+9vrFq1yna8tLTUGDdunNG8eXPD3d3dCAgIMObMmWM7V5JtGz58uGEYhvHdd98ZDz/8sOHl5WU0aNDAeOyxx4yCggKH/q9Zs8YIDAw0LBaLUV5ebkgyVq5caTz44IOGp6enERQUZGRnZxsHDx407r77bqNevXpG7969ja+++spuDNu2bTN69uxpuLu7G4GBgcb06dONc+fO2Y5LMpKTk42HH37YqFevnvHSSy9dzT+SGxKPl8Np6tevr/r16+utt95SaWmpw/EdO3bo2LFj+sMf/uBwrH///rr55puVmpp6TfoCADVV8+bNZbVatXv37t+sGx0draysLNusdnp6utq2bauePXtWeo7FYtH69ev14YcfaunSpZKksWPHys/Pz/a45muvvaYpU6Zo9uzZOnDggObMmaNp06Zp/fr1dm298MILmjBhgg4cOKB+/fpd4YgB4MpMmjRJiYmJOnDggLp3765Tp07pgQce0M6dO5Wbm6t+/fqpf//+dk/+SNLChQvVq1cv5ebmKjY2Vs8884z+/e9/S5KWLl2qbdu2adOmTfryyy/15z//WW3btpUkffTRR7r//vs1ZMgQWa1WLVmyRIZhaODAgfrvf/+r9957T5mZmfr6668VFRVld82vvvpKmzZtUnp6uvLy8mzls2bN0rBhw5SXl6egoCA9/vjjGjNmjBISErRv3z5J0vjx4231t2/frieffFITJkzQ/v37tWrVKqWkpGj27Nl213v55Zc1YMAAffbZZxo5cmR1feQ3DmenftzY3nzzTaNRo0aGh4eHERoaaiQkJBiffPKJYRiGMXfuXEOS8dNPP1V47sMPP2x06tTJti/J8PDwMLy8vGxbRTPfqmRG/FJ9AYDrXWUz3efPnzdGjBhhSDKaN29uDBw40Fi2bJlRXFxsq3Nxluenn34yBg4caMyYMcMwDMO49957jSVLlhhbtmypdKb7ok2bNhnu7u5GQkKCUa9ePePLL7+0HfP39zfeeOMNu/qzZs0yQkJCDMP4/zPdSUlJV/sxAICdqsx0v/XWW7/ZXufOnY1ly5bZ9gMCAownn3zStl9eXm40a9bMSE5ONgzDMJ599lnjd7/7ne1JoF8bMGCAbYbbMAxjx44dhqurq5Gfn28r++KLLwxJxocffmjrf926dY3CwkK7tvR/TyxdtHfvXkOSsWbNGltZamqq4eHhYdsPCwuzzbxf9PrrrxstWrSwazcuLq7yDwW/iZluONXgwYP1ww8/aNu2berXr5927dqlnj17KiUlxVbHMIwKzzUMw+773NKFl1Hk5eXZtvDw8GrtCwDUNK6urlq3bp2+//57zZs3Ty1bttTs2bPVpUsXu+84XjRy5EilpKTom2++0d69e/XEE09c1nUee+wxPfLII0pMTNTChQt18803S5KOHj2qw4cPKyYmxvZUUf369fXKK6/o66+/tmujV69eVz9gALhCv74HnT59WpMmTVLnzp3VsGFD1a9fX//+978dZrq7d+9u+9lisah58+YqLCyUJI0YMUJ5eXnq2LGjJkyYoB07dlyyDwcOHJC/v7/8/f1tZRevf+DAAVtZQECAmjZt6nD+L/vi5+cnSerWrZtd2ZkzZ3TixAlJUk5OjmbOnGl3f3766adltVpVUlJS6WeDqiF0w+k8PDwUHh6ul156SdnZ2RoxYoRefvll2y9sv7zB/NK///1vh5dONG/eXO3bt7dtXl5e1dIXAKjpWrVqpejoaC1fvlz79+/XmTNntHLlSod6DzzwgM6cOaOYmBj1799fTZo0uaz2S0pKlJOTI1dXVx08eNBWfvFlba+99prdH0U///xzvf/++3ZtVPWeDQDV6df3oOeff17p6emaPXu2srKylJeXp27duuns2bN29X79UjGLxWK79/Xs2VOHDh3SrFmz9PPPP2vIkCF69NFHK+1DRZNKFZVXdr/8ZV8u1q+o7GL/ysvLNWPGDLv782effaaDBw/Kw8PjN6+Hy0PoxnWnc+fOOn36tCIiItS4cWMtXLjQoc62bdt08OBBuzfsmtkXAKhNGjVqpBYtWlR4f3N1dVV0dLR27dpVpe/t/c///I9cXFz097//XUuXLtU///lPSRdmVVq1aqVvvvnG7o+i7du3V2BgYLWNCQCqW1ZWlkaMGKFBgwapW7duat68ub799tsqt+Pt7a2oqCi99tprSktLU3p6eqWr43Tu3Fn5+fk6fPiwrWz//v0qLi5Wp06drnQolerZs6e+/PJLh/tz+/bt5eJCVKwuLBkGpzl27Jgee+wxjRw5Ut27d1eDBg20b98+zZs3TwMGDJCXl5dWrVql3//+9xo9erTGjx8vb29vvfPOO3r++ef16KOPasiQIdekLwBQU61atUp5eXkaNGiQ2rVrpzNnzmjDhg364osvtGzZsgrPmTVrlp5//vnLnuX+29/+prVr12rv3r3q2bOnJk+erOHDh+vTTz9Vo0aNNH36dE2YMEHe3t6KjIxUaWmp9u3bp59++knx8fHVOVwAqDbt27fX5s2b1b9/f1ksFk2bNs1hqcXfsnjxYrVo0UK33nqrXFxc9Je//EXNmzdXw4YNK6zft29fde/eXU888YSSkpJ0/vx5xcbG6u677zblEe+XXnpJDz30kPz9/fXYY4/JxcVFn376qT777DO98sor1X69GxWhG05Tv3593XHHHVq8eLG+/vprnTt3Tv7+/nr66af14osvSpIeffRRvfvuu5ozZ47uuusu/fzzz2rfvr2mTJmiuLi4Ch+/MasvAFAT3X777dqzZ4/Gjh2rH374QfXr11eXLl301ltv6e67767wHDc3N/n6+l5W+0ePHlVMTIymT59ue8v5yy+/rB07dmjs2LFKS0vTqFGjVK9ePc2fP1+TJk2Sl5eXunXrpri4uOoaJgBUu8WLF2vkyJEKDQ2Vr6+vXnjhBdt3oS9X/fr19eqrr+rgwYNydXXVbbfdpoyMjEpnkS0Wi9566y09++yzuuuuu+Ti4qL777+/0j+SXq1+/frpr3/9q2bOnKl58+apbt26CgoK0qhRo0y53o3KYlT2lioAAAAAAHBVeFAfAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAFy2Xbt2yWKx6Pjx45d9Ttu2bZWUlGRanwAAuJ4RugEAqEVGjBghi8WisWPHOhyLjY2VxWLRiBEjrn3HAAC4QRG6AQCoZfz9/bVx40b9/PPPtrIzZ84oNTVVbdq0cWLPAAC48RC6AQCoZXr27Kk2bdpo8+bNtrLNmzfL399fPXr0sJWVlpZqwoQJatasmTw8PHTnnXfqo48+smsrIyNDN998szw9PXXvvffq22+/dbhedna27rrrLnl6esrf318TJkzQ6dOnK+3f9OnT1aZNG7m7u6tly5aaMGHC1Q8aAIDrFKEbAIBa6KmnntK6dets+2vXrtXIkSPt6kyaNEnp6elav369Pv74Y7Vv3179+vXTf//7X0nS4cOH9cgjj+iBBx5QXl6eRo0apcmTJ9u18dlnn6lfv3565JFH9OmnnyotLU179uzR+PHjK+zXm2++qcWLF2vVqlU6ePCg3nrrLXXr1q2aRw8AwPWD0A0AQC0UHR2tPXv26Ntvv9V3332nf/3rX3ryySdtx0+fPq3k5GTNnz9fkZGR6ty5s1577TV5enpqzZo1kqTk5GTddNNNWrx4sTp27KgnnnjC4fvg8+fP1+OPP664uDh16NBBoaGhWrp0qTZs2KAzZ8449Cs/P1/NmzdX37591aZNG91+++16+umnTf0sAABwJkI3AAC1kK+vrx588EGtX79e69at04MPPihfX1/b8a+//lrnzp1Tnz59bGV169bV7bffrgMHDkiSDhw4oN69e8tisdjqhISE2F0nJydHKSkpql+/vm3r16+fysvLdejQIYd+PfbYY/r5559100036emnn9aWLVt0/vz56h4+AADXjTrO7gAAADDHyJEjbY95L1++3O6YYRiSZBeoL5ZfLLtY51LKy8s1ZsyYCr+XXdFL2/z9/fXll18qMzNTO3fuVGxsrObPn6/33ntPdevWvbyBAQBQgzDTDQBALXX//ffr7NmzOnv2rPr162d3rH379nJzc9OePXtsZefOndO+ffvUqVMnSVLnzp31/vvv25336/2ePXvqiy++UPv27R02Nze3Cvvl6emphx9+WEuXLtWuXbu0d+9effbZZ9UxZAAArjvMdAMAUEu5urraHhV3dXW1O+bl5aVnnnlGzz//vBo3bqw2bdpo3rx5KikpUUxMjCRp7NixWrhwoeLj4zVmzBjbo+S/9MILL6h3794aN26cnn76aXl5eenAgQPKzMzUsmXLHPqUkpKisrIy3XHHHapXr55ef/11eXp6KiAgwJwPAQAAJ2OmGwCAWszb21ve3t4VHps7d64GDx6s6Oho9ezZU1999ZW2b9+uRo0aSbrweHh6errefvtt3XLLLVq5cqXmzJlj10b37t313nvv6eDBgwoLC1OPHj00bdo0tWjRosJrNmzYUK+99pr69Omj7t2765133tHbb7+tJk2aVO/AAQC4TliMy/nCFgAAAAAAqDJmugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJMQugEAAAAAMAmhGwAAAAAAkxC6AQAAAAAwCaEbAAAAAACTELoBAAAAADAJoRsAAAAAAJP8Pxe0IJ85dI3uAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics = average_metrics['metric']\n",
    "models = average_metrics.columns[1:]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "bar_width = 0.2\n",
    "\n",
    "r1 = range(len(models))\n",
    "r2 = [x + bar_width for x in r1]\n",
    "\n",
    "ax.bar(r1, average_metrics[metrics_df['metric'] == 'mae'].values[0][1:], color='b', width=bar_width, edgecolor='grey', label='mae')\n",
    "ax.bar(r2, average_metrics[metrics_df['metric'] == 'mse'].values[0][1:], color='g', width=bar_width, edgecolor='grey', label='mse')\n",
    "\n",
    "for i, v in enumerate(average_metrics[average_metrics['metric'] == 'mae'].values[0][1:]):\n",
    "    ax.text(i - 0.05, v + 0.001, str(round(v, 3)))\n",
    "for i, v in enumerate(average_metrics[average_metrics['metric'] == 'mse'].values[0][1:]):\n",
    "    ax.text(i + bar_width - 0.05, v + 0.001, str(round(v, 3)))\n",
    "\n",
    "ax.set_xlabel('Models')\n",
    "ax.set_ylabel('Values')\n",
    "\n",
    "ax.set_xticks([r + bar_width/2 for r in range(len(models))])\n",
    "ax.set_xticklabels(models)\n",
    "\n",
    "ax.legend()\n",
    "\n",
    "plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "neuralforecast",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}