diff --git a/CHANGELOG.md b/CHANGELOG.md index 42287d4..f853bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ assert is_masked(masked_tree[0]) is True ``` +- Add `dataclasses` rule for `tree_{repr,str}` + ## V0.12 ### Deprecations diff --git a/docs/index.rst b/docs/index.rst index 4a0b89e..caf145a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,6 @@ Install from pip:: :maxdepth: 1 notebooks/[guides]surgery - notebooks/[guides]optimlib .. toctree:: :caption: API Documentation diff --git a/docs/notebooks/[guides]optimlib.ipynb b/docs/notebooks/[guides]optimlib.ipynb deleted file mode 100644 index 6a393bd..0000000 --- a/docs/notebooks/[guides]optimlib.ipynb +++ /dev/null @@ -1,527 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 🧮 Mini optimizer library" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the following example a mini optimizer library is built using `sepes`. The strategy will be to write the optimizer methods with the inplace modification as done in similar libraries like `PyTorch`, then use `value_and_tree` to execute the inplace modification on a new instance and comply with `jax` functional updates." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install sepes" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import jax\n", - "import jax.numpy as jnp\n", - "import jax.random as jr\n", - "import sepes as sp\n", - "import functools as ft\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## MLP" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "class MLP(sp.TreeClass):\n", - " def __init__(self, *, key: jax.Array):\n", - " k1, k2 = jr.split(key)\n", - " self.w1 = jax.random.normal(k1, [10, 1])\n", - " self.b1 = jnp.zeros([10], dtype=jnp.float32)\n", - " self.w2 = jax.random.normal(k2, [1, 10])\n", - " self.b2 = jnp.zeros([1], dtype=jnp.float32)\n", - "\n", - " def __call__(self, input: jax.Array) -> jax.Array:\n", - " output = input @ self.w1.T + self.b1\n", - " output = jax.nn.relu(output)\n", - " output = output @ self.w2.T + self.b2\n", - " return output\n", - "\n", - "\n", - "def loss_func(net: MLP, input: jax.Array, target: jax.Array) -> jax.Array:\n", - " return jnp.mean((net(input) - target) ** 2)\n", - "\n", - "\n", - "input = jnp.linspace(-1, 1, 100).reshape(-1, 1)\n", - "target = input**2 + 0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## First-order optimization" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Optimizer (Adam)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def moment_update(grads, moments, *, beta: float, order: int):\n", - " def moment_step(grad, moment):\n", - " return beta * moment + (1 - beta) * (grad**order)\n", - "\n", - " return jax.tree_map(moment_step, grads, moments)\n", - "\n", - "\n", - "def debias_update(moments, *, beta: float, count: int):\n", - " def debias_step(moment):\n", - " return moment / (1 - beta**count)\n", - "\n", - " return jax.tree_map(debias_step, moments)\n", - "\n", - "\n", - "class Adam(sp.TreeClass):\n", - " \"\"\"Apply the Adam update rule to the incoming updates\n", - "\n", - " Args:\n", - " tree: PyTree of parameters to be optimized\n", - " beta1: exponential decay rate for the first moment\n", - " beta2: exponential decay rate for the second moment\n", - " eps: small value to avoid division by zero\n", - "\n", - " Note:\n", - " This implementation does not scale the updates by the learning rate.\n", - " Use ``jax.tree_map(lambda x: x * lr, updates)`` to scale the updates.\n", - " \"\"\"\n", - "\n", - " def __init__(\n", - " self,\n", - " tree,\n", - " beta1: float = 0.9,\n", - " beta2: float = 0.999,\n", - " eps: float = 1e-8,\n", - " ):\n", - " self.beta1 = beta1\n", - " self.beta2 = beta2\n", - " self.eps = eps\n", - " self.mu = jax.tree_map(jnp.zeros_like, tree)\n", - " self.nu = jax.tree_map(jnp.zeros_like, tree)\n", - " self.count = 0\n", - "\n", - " def __call__(self, updates):\n", - " \"\"\"Apply the Adam update rule to the incoming updates\"\"\"\n", - " # this method will transform the incoming updates(gradients) into\n", - " # the updates that will be applied to the parameters\n", - "\n", - " # NOTE: calling this method will raise `AttributeError` because\n", - " # its mutating the state (e.g. self.something=something)\n", - " # it will only work if used with `value_and_tree` that executes it functionally\n", - " self.count += 1\n", - " self.mu = moment_update(updates, self.mu, beta=self.beta1, order=1)\n", - " self.nu = moment_update(updates, self.nu, beta=self.beta2, order=2)\n", - " mu_hat = debias_update(self.mu, beta=self.beta1, count=self.count)\n", - " nu_hat = debias_update(self.nu, beta=self.beta2, count=self.count)\n", - "\n", - " def update(mu, nu):\n", - " return mu / (jnp.sqrt(nu) + self.eps)\n", - "\n", - " return jax.tree_map(update, mu_hat, nu_hat)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Learning rate scheduler (Exponential decay)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "class ExponentialDecay(sp.TreeClass):\n", - " \"\"\"Scale the incoming updates by an exponentially decaying learning rate\n", - "\n", - " Args:\n", - " init_rate: initial learning rate\n", - " decay_rate: rate of decay\n", - " transition_steps: number of steps to transition from init_rate to 0\n", - " transition_begins: number of steps to wait before starting the transition\n", - " \"\"\"\n", - "\n", - " def __init__(\n", - " self,\n", - " init_rate: float,\n", - " *,\n", - " decay_rate: float,\n", - " transition_steps: int,\n", - " transition_begins: int = 0,\n", - " ):\n", - " self.count = 0\n", - " self.rate = init_rate\n", - " self.init_rate = init_rate\n", - " self.decay_rate = decay_rate\n", - " self.transition_begins = transition_begins\n", - " self.transition_steps = transition_steps\n", - "\n", - " def __call__(self, updates):\n", - " \"\"\"Scale the updates by the current learning rate\"\"\"\n", - " # NOTE: calling this method will raise `AttributeError` because\n", - " # its mutating the state (e.g. self.something=something)\n", - " # it will only work if used with `value_and_tree` that executes it functionally\n", - " self.count += 1\n", - " count = self.count - self.transition_begins\n", - " self.rate = jnp.where(\n", - " count <= 0,\n", - " self.init_rate,\n", - " self.init_rate * self.decay_rate ** (count / self.transition_steps),\n", - " )\n", - " return jax.tree_map(lambda x: x * self.rate, updates)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Composing" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "class Optim(sp.TreeClass):\n", - " def __init__(self, net):\n", - " self.adam = Adam(net)\n", - " self.lr = ExponentialDecay(-1e-3, decay_rate=0.99, transition_steps=1000)\n", - "\n", - " def __call__(self, updates):\n", - " updates = self.adam(updates)\n", - " updates = self.lr(updates)\n", - " return updates" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch=1000\tLoss: 2.438e-02\n", - "Epoch=2000\tLoss: 4.649e-03\n", - "Epoch=3000\tLoss: 2.498e-03\n", - "Epoch=4000\tLoss: 1.100e-03\n", - "Epoch=5000\tLoss: 6.216e-04\n", - "Epoch=6000\tLoss: 4.409e-04\n", - "Epoch=7000\tLoss: 3.273e-04\n", - "Epoch=8000\tLoss: 2.752e-04\n", - "Epoch=9000\tLoss: 2.299e-04\n", - "Epoch=10000\tLoss: 1.949e-04\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmxElEQVR4nO3dd1yVdf/H8dcZHJaAExBF3HuLe5u5MrWlpeVub+u+77zb4/5Z992+KxvOysyGZqVllpqWG7e4Fw4QURmCrHOu3x8HuSPRBIELDu/n43EewvdcB94XF8iH7/UdFsMwDERERERMYjU7gIiIiJRvKkZERETEVCpGRERExFQqRkRERMRUKkZERETEVCpGRERExFQqRkRERMRUKkZERETEVHazA1wJl8vFiRMnCAgIwGKxmB1HREREroBhGKSkpBAWFobVeun+jzJRjJw4cYLw8HCzY4iIiEghHD16lJo1a17y+TJRjAQEBADukwkMDDQ5jYiIiFyJ5ORkwsPDc3+PX0qZKEYu3JoJDAxUMSIiIlLG/NUQCw1gFREREVOpGBERERFTqRgRERERU6kYEREREVOpGBERERFTqRgRERERU6kYEREREVOpGBERERFTqRgRERERU6kYEREREVOpGBERERFTqRgRERERU5XrYuTn6JPc/9km9sefMzuKiIiIKeauj+HF76PZezLFtAzluhj5bH0Mi7bF8t3WE2ZHERERMcUna44w/bdDRB05a1qGcl2MXN+qOgDfbzuBYRgmpxERESlZB06dIzo2GbvVwoBmoablKNfFSN8mIXjbrRw4lcquWPO6p0RERMzw/dZYALo3qEolf4dpOcp1MRLg40WfxsEAfLdNt2pERKT8MAyDb7ceB+D6VmGmZinXxQjA4JbuC/DdVt2qERGR8mN3XAoHTqXisFu5tmmIqVnKfTHSp3Ewfg4bx86eZ8vRRLPjiIiIlIgLkzd6N6pGgI+XqVnKfTHi67DlVoTfb4s1OY2IiEjxMwwjd3iC2bdoQMUIANfn3Kr5ftsJXC7dqhEREc+29VgSR8+cx89hyx07aabyXYy4nLB7Ed3rBhDoY+dkcgYbDp8xO5WIiEix+j7nFk3fJiH4OewmpynvxciswfD5SLx3L2RAc/f8as2qERERT+ZyGbnDEkrDLRoo78VIg77uf9e9z+AW7gXQftgeR7bTZWIoERGR4rPxyFniktMJ8LHTo2FVOLIadswHZ7Zpmcp3MdJ2LNh9IHYLXbwPUNnfwenUTNYcPG12MhERkWJxYRZN/2aheNttsPz/4KtxsOo10zKV72LEvwq0uAUA+4YPGdQi51aN9qoREREPlO10sXj7H27RxG6Dw6vAYoM2o0zLVb6LEYCOd7v/jV7ITfUsAPy4I46MbKeJoURERIremoOnOZ2aSWV/B13qVYG177mfaDoUgmqalkvFSGgLiOgGhpPWJ+cTGuhDcno2v+45ZXYyERGRIrVwi7vnf2DzULzS4mH7V+4nOt9vYioVI245vSOWqJnc0KISAN/qVo2IiHiQ9CwnS3bEATC0dQ3YMA1cWRDeEWpGmppNxQhAo0EQVAvOn2GUfxQAP+86ybkM80YWi4iIFKUVe+JJycimepAPkWE+sHGG+4lO95kbDBUjbjY7dJgIQI29s6lbxY/0LBdLo+NMDiYiIlI0LvT4D2kVhnXHF5B22v2HeOPBJidTMfI/be4Auy+WuO3cUzcegG+36FaNiIiUfSnpWfy8y/27bUir6rB2qvuJjne7/yA3mYqRC/wqQ6sRAAxK+waAlfsSOH0uw8RQIiIiV2/JzpNkZruoV82fpmkb4dRucFSAtneYHQ1QMZJXx3sAqHDwR/qEZuB0GSzeoVs1IiJStl24RTO0dQ0sF6bztrkDfIJMTPU/Kkb+KLgJ1OkJhouHAn8F4DvdqhERkTIs4VwGv+9PAOCm8HNw4BfA8r91tkoBFSN/ltM70jJ+Ib6WDNYfPsPxxPMmhxIRESmcxdtjcboMWtUMosbume7GxtdB5TrmBvsDFSN/1rA/VKqNNf0sjwRvAbQ8vIiIlF0XFjq7uYkvbJvnbjR5kbM/UzHyZ1YbdLgLgBHORYChWTUiIlImHT2TRtSRs1gscINzCWSnQ/XWUKuz2dHyUDGSn9ajwMufiuf20822i+jYZPadTDE7lYiISIF8t839x3S32gFU2DbL3dj5frBYzAuVDxUj+fGtCK1vA+CxwGXA/7q5REREyooLPfv3V9sG505CQHVoOszcUPlQMXIpHdyjjFufX0NNSzwLtx7HMAyTQ4mIiFyZXbHJ7I5LwWGz0D7uc3djhzvB7jA3WD5UjFxKtYZQ7xosGExw/MzRM+fZFHPW7FQiIiJX5JstxwG4O+IEtvjtYPeFduNMTpU/FSOXkzPN91bbCvxI55vNulUjIiKln8v1v8kXtxuL3I2tb3OvNl4KFbgYWblyJddffz1hYWFYLBa++eabv3zNr7/+Srt27fDx8aFu3bq8//77hcla8ur3hcr18HWd40bbKr7fdoIsp8vsVCIiIpe17tAZYpPSaeqTQHDscndjKdid91IKXIykpqbSqlUr3nnnnSs6/tChQwwaNIju3buzefNm/vnPf/LQQw/x9ddfFzhsibNac1eom+D1E2fTMlm595TJoURERC5vYc4tmqeqrMCCAQ36QdUG5oa6jAJv1Tdw4EAGDhx4xce///771KpVizfffBOAJk2asHHjRl599VVuuummgn76ktfqNvjlRepkHqebdQcLNodxTZMQs1OJiIjkKz3LyaLtsQSSSsekH92NpbhXBEpgzMiaNWvo169fnrb+/fuzceNGsrKy8n1NRkYGycnJeR6m8QmENqMAGGv7kaXRJ0lJzz+3iIiI2VbsiSclPZu7/Fdhy06D4GZQt5fZsS6r2IuRuLg4QkLy9iSEhISQnZ1NQkJCvq+ZMmUKQUFBuY/w8PDijnl5He7CwEIf2xZCnSdYsvOkuXlEREQuYcHm49jJ5g7bEndDp3tL3SJnf1Yis2ksf/oiXFiv48/tF0yePJmkpKTcx9GjR4s942VVqYelQT+sGIyx/cQ3m4+bm0dERCQfSWlZLN99ioHW9QRlngT/atDiFrNj/aViL0ZCQ0OJi4vL0xYfH4/dbqdKlSr5vsbb25vAwMA8D9PlDGS9xfYrWw8c5WRyusmBRERE8lq8I5ZMp5P7fX9yN0ROAC8fc0NdgWIvRjp37szSpUvztP30009ERkbi5eVV3J++6NTrA1UbEmA5zw3WldrJV0RESp0Fm4/T1rKPxs69YHNA+wlmR7oiBS5Gzp07x5YtW9iyZQvgnrq7ZcsWYmJiAPctltGjR+cef88993DkyBEmTZrErl27mDFjBtOnT+fxxx8vmjMoKRZL7m6+Y2w/sXCzybeORERE/uB44nnWHzrDRPtid0PL4VAh2NxQV6jAxcjGjRtp06YNbdq0AWDSpEm0adOGZ555BoDY2NjcwgSgTp06LF68mBUrVtC6dWtefPFF3n777bIxrffPWt2GyzuQetZYqsT9pp18RUSk1Fi45Tg1LacYYNvobijl03n/yGKUgd3fkpOTCQoKIikpyfzxI0uehDXvsMLZinVdP+QfAxqbm0dERMo9wzC49o2VjDjzPnfaF7un8o5eaHasK/79rb1pCqr9RAws9LJtZVPUelyuUl/LiYiIh9txPJnY+FPcaruw9Pv95gYqIBUjBVW5Dq6GAwAYeP471h48bXIgEREp777edIzhthUEWM5DlQbuvdXKEBUjhWDr5N7N92bbShZt2GNyGhERKc+ynC4WbTnKONuFpd/vde+tVoaUrbSlRZ2enK/YkAqWdAJ2fU5aZrbZiUREpJxaufcUbdPXUst6CsO3kntPtTJGxUhhWCz4dHOPUr6NH/lph9YcERERc8zfdJzx9h8AsLQbBw4/kxMVnIqRQrK0HE66PZAIazwH13xjdhwRESmHks5ncWLXGjpad2NY7NDhTrMjFYqKkcJy+JPRwr2bb/uTXxCv5eFFRKSELd4ey2jLIvc7zW+EwDBzAxWSipGrENTzPpxY6W7dzq+/rzI7joiIlDMrNmxlsHUtAJbOZWeRsz9TMXI1KtbieEhvAHw3Tzc5jIiIlCcxp9NoGfsFXhYnmTU6QVgbsyMVmoqRq1Sp94MA9MlYxu5DMX9xtIiISNH4buMBRtqWAeDo9oDJaa6OipGrFNCoF8ccdfGzZHDslw/MjiMiIuWAYRhkRH1KJcs5zvmFQ6NBZke6KipGrpbFQlLL8QA0OfYF2VlZJgcSERFPt+nIaYamu/ee8epyL1htJie6OipGikDDvuNJJIAaxBP96zyz44iIiIfbvuJr6lljOW/1xzvyDrPjXDUVI0XAy8ef7aE3AOAd9ZHJaURExJOlZzlpdOgTAM42uhV8TN7NvgioGCkiwX3uI9uw0uj8FpKPbDE7joiIeKjVq1fS2bIdJ1ZCr33I7DhFQsVIEWnUsAm/O7oCcPKnt0xOIyIinsq2fioAB6r0xlq5trlhioiKkSKUnDOQtdbx7yHtjMlpRETE08THHqXTOfd03sDentErAipGilTnXoPY4aqDN5kkrPzQ7DgiIuJhDi/5L96WLPZ5NSK0WU+z4xQZFSNFqGqAD+uDbwHAa9MMcGabnEhERDyFkXWe+kc+ByCh+QSwWExOVHRUjBSxGt1v55QRSFDmSZy7vjc7joiIeIiYXz+mspFErFGZ5n3L/nTeP1IxUsR6NwtngeVaAFJ+/a/JaURExCMYBt4b3at8bwy5hQB/P5MDFS0VI0XMYbeS3PwOsgwbFU9thNitZkcSEZEyLnPfckLTD5BmeFO1x91mxylyKkaKQf9ObVns6ghA5ur3TU4jIiJl3dllbwKw2NaHDk3rmhumGKgYKQbNawTyS+CNANh2fgWpCSYnEhGRMithHyFxv+IyLJxtOR6b1XMGrl6gYqQYWCwWWna6hi2uethcmRA10+xIIiJSRqWudI8//MXVhmu7dTU5TfFQMVJMhrWpwceuAQBkrf0InNrNV0RECijtDI4d7g1YV1cbQe2q/iYHKh4qRopJ1QrepDe8nnijIl5pJ2HXt2ZHEhGRMsaImoWXK51oVwTNulxndpxio2KkGN3Uvi6fZvcFwLV2qslpRESkTHFmkbXGPZ33U65jUMvqJgcqPipGilHPhtVY4juQTMOG9dgGOB5ldiQRESkrdn6DIy2OU0YQlhY34eewm52o2KgYKUZ2m5Ve7Zrxnauzu2Gd9qsREZErYBg4V78DwMfZ13JjB8+bzvtHKkaK2S3twpmd3R8AY8fXkHLS5EQiIlLqxazFFreFDMOL3yoOoW2tSmYnKlYqRopZ/eAKeNWKJMrVAIsrS9N8RUTkr619F4Cvnd3o36E5Fg/aFC8/KkZKwPDImszMdk/zNTZMh+xMkxOJiEipdfYwxu5FAMx2DeLGNjVMDlT8VIyUgOtahvGrrRNxRiUsqfEQ/Y3ZkUREpLRa9wEWw8WvzpbUbNiG4EAfsxMVOxUjJaCCt51+LcL5JNu9my/rtF+NiIjkIz0ZY9MnAEx3DuSWyHCTA5UMFSMlZHhkTeY6+5BheLmn+B7dYHYkEREpbTZ/giUzhb2uGuz0iaRP42CzE5UIFSMlpEOdygRVrc63zgvTfNU7IiIif+DMhrXu3w0znAO5KTIch718/JouH2dZClgsFka0D2eW0z2QlehvIDnW1EwiIlKK7P4ekmI4bQSwwNmN4eXkFg2oGClRN7atwR5LHda5GoMrGzbOMDuSiIiUFmvfA2CO8xpa1g6hfnAFkwOVHBUjJSg4wIdrmgQzK2cRNDbOgKx0c0OJiIj5jkXB0XVkYeeT7Gu5tX0tsxOVKBUjJezW9rX4yRVJLFUgLQF2zjc7koiImC1nkbNvnZ1J967GoBaeuyleflSMlLAeDatRLdCfj7NypvmunQqGYW4oERExT9Ix2PkNANOzBzK0TRi+Dpu5mUqYipESZrNacqb59iYTB8Rtg6PrzI4lIiJmWf8hGE7WupoSbdQud7doQMWIKW6JDCfJEsDX2V3dDWunmhtIRETMkXEOomYBMC17IM3CAmleI8jcTCZQMWKC8Mp+dKtfldnOnIGsu75zd9OJiEj5snUupCdx3FqdX1xtuLV9+ZnO+0cqRkwyon04u41aRFmag+GEDdPMjiQiIiXJ5crtGf8gox8Ou50hrT1/U7z8qBgxybVNQ6jk58WHGTkDWaNmQ9Z5c0OJiEjJ2bcEzhzgvLUCXzl7cl2L6gT5epmdyhQqRkzibbdxU9uaLHW1I8EeAufPwPYvzY4lIiIlZY17Ou9nzt6k4cOIcnqLBlSMmOrWDrVwYeXD9GvcDes+0DRfEZHyIHYbHF6Fy2JjWkY/6gdXoEOdymanMo2KERPVD65AxzqV+Ty7F1lWHzi5A478bnYsEREpbjljRVbZuxBLFW7rUAuLxWJyKPOoGDHZyI61SKYC39HT3aDdfEVEPFvKSdjxFQBvnLsWh93KjW3K58DVC1SMmKx/s1Aq+Xnx3vmcWzW7F0FijLmhRESk+GyYBs5Mjvg1Z4tRn0HNQ6nk7zA7lalUjJjMx8s9kHW/UZNon7ZguDTNV0TEU2Wdh43TAXjzXF8ARnaMMDNRqaBipBS4raN76d/XU3q7G6JmQ2aqiYlERKRYbPsC0k5zzqc632a2o141f9rXrmR2KtOpGCkF6lVzD2T9xdmGRJ+akJ7o/oYVERHPYRi5A1fnWgbixFbuB65eUKhi5L333qNOnTr4+PjQrl07Vq1addnj58yZQ6tWrfDz86N69eqMGzeO06dPFyqwpxrZsRYGVmZd2M1X03xFRDzLgWVwahdOL3/ePtsFh93KTW1rmp2qVChwMTJv3jweeeQRnnzySTZv3kz37t0ZOHAgMTH5D7r87bffGD16NBMmTGDnzp18+eWXbNiwgYkTJ151eE9yYSDr9NSuZNv94NQuOLTS7FgiIlJU1r7n/idwICn4aeDqHxS4GHn99deZMGECEydOpEmTJrz55puEh4czdWr+O8+uXbuW2rVr89BDD1GnTh26devG3XffzcaNG686vCe5MJA1BT9+9b3QO6JpviIiHuHUHtj/MwYWXjjVHdDA1T8qUDGSmZlJVFQU/fr1y9Per18/Vq9ene9runTpwrFjx1i8eDGGYXDy5Em++uorrrvuukt+noyMDJKTk/M8yoMLA1n/77T7G5U9P8CZQyYmEhGRIpHTK3I0uBd7Mqtp4OqfFKgYSUhIwOl0EhISkqc9JCSEuLi4fF/TpUsX5syZw4gRI3A4HISGhlKxYkX++9//XvLzTJkyhaCgoNxHeHj5WK+/XrUKdKlXhQOuMA5V7AwYmuYrIlLWpZ6GrZ8D8Faq+4/52ztFaODqHxRqAOufv4CGYVzyixodHc1DDz3EM888Q1RUFD/++COHDh3innvuueTHnzx5MklJSbmPo0ePFiZmmTQqp9vujeQ+7oZNn0DGORMTiYjIVYmaAdnppFZpztena+HjZeVGDVzNw16Qg6tWrYrNZruoFyQ+Pv6i3pILpkyZQteuXfnb3/4GQMuWLfH396d79+689NJLVK9e/aLXeHt74+3tXZBoHqNfsxCqBXjzXUoT/q9qBBXOHYGtc6HDnWZHExGRgsrOhPXuHu75jiGAhSGtwgjy9TI3VylToJ4Rh8NBu3btWLp0aZ72pUuX0qVLl3xfk5aWhtWa99PYbDbA3aMieXnZrNzaPhwDK19aB7kb138ILpe5wUREpOB2zodzcbj8Q3g5pingvkUjeRX4Ns2kSZOYNm0aM2bMYNeuXTz66KPExMTk3naZPHkyo0ePzj3++uuvZ/78+UydOpWDBw/y+++/89BDD9GhQwfCwsKK7kw8yG0damG1wGvx7XB5VYCEvXBwmdmxRESkIAwD1rwLwIbgm0l1WmlZM4iWNSuam6sUKtBtGoARI0Zw+vRpXnjhBWJjY2nevDmLFy8mIsJd6cXGxuZZc2Ts2LGkpKTwzjvv8Nhjj1GxYkX69OnDK6+8UnRn4WHCKvrSp3EIP+86yfqKA+l06ktY9yHU72t2NBERuVJHfoe4bRh2X16I6wDA7ZrOmy+LUQbulSQnJxMUFERSUhKBgYFmxykRK/bEM3bmBpr6JLCIh7FgwIOboEo9s6OJiMiVmDsS9izieP1b6bpjCIE+dtb9sy++DpvZyUrMlf7+1t40pVSPBtWoVdmP6PSqxAbnrDuy/kNzQ4mIyJU5fQD2LAZg6nn3dN6b2tUsV4VIQagYKaWsVgsjcxZBe/98zoqsm+dAevlYAE5EpExb9wFgkF67D58d9AH+t3SDXEzFSCl2S7uaOGxWPj5Vl/SgepCZ4p7mKyIipdf5RNj8KQDf+92Ay4DOdatQP7iCublKMRUjpViVCt4MahEKWFjkN9TduO4DTfMVESnNNs2GrFRc1Zrw8h73Wlqaznt5KkZKuTs61wbgxaMtcXkHwpkDsP9nc0OJiEj+nNnu2Y/A1hojSUjNJCTQm37N8l8YVNxUjJRybWtVpHmNQBKzHWwPHuJu1G6+IiKl066FkHwM/KryyomWAIzsEIGXTb9uL0dfnVLOYrEwulNtAF462RUDCxz4BU7tNTeYiIhcbI17d974xqNYG5OK3Wrhtg7lY7PXq6FipAwY0jqMin5ebEgOIiEsZwO99R+YG0pERPI6uh6ObwSbgw/S3P9XD2xRneBAH5ODlX4qRsoAHy8bwyPdlfW0LPd8dbbMdY/YFhGR0iFn6feMpjfx6Y7zAIzurIGrV0LFSBlxe8cILBb44GhNMis3gqxU2DLH7FgiIgKQGAO7vgXgO98byMh20aR6IJERlUwOVjaoGCkjalXxo0+jYMDCkgo503zXfwgup6m5REQE97ILhgujTk/e3uEAYEznCCwWi8nBygYVI2XIHTndfS/EtMDwqQhnD8O+n0zNJCJS7mWkwKaPAdgePoqYM2kE+tgZ2rqGycHKDhUjZUiPBtWoXcWPU+k2oqvf4G5cO9XcUCIi5d3mOZCRDFUa8MZh9x+NwyPDtQ9NAagYKUOsVkvuKn4vJ3TDsFjh0K8Qv8vkZCIi5ZTLCevcfxSebj6eFftOA1pxtaBUjJQxt0SG4+tlY9UpX86E52ygt07TfEVETLHnB/ctc5+KfJjUEcOAng2rUbuqv9nJyhQVI2VMkK8XN7Z134ecmd3f3bj1czh/1sRUIiLl1Fr3ImeZbcbw2eYEAMZ1rW1ioLJJxUgZNLZLbQDeOxRCZtWmkH0+d/CUiIiUkBNb4MjvYLWz0Os6UjKyqVvVnx4NqpmdrMxRMVIGNQgJoHuDqrgMCz8F5AxkXf+Re4MmEREpGTm9IkbTG5i6yb3I2diutbFaNZ23oFSMlFEXekeeP9QEw7cyJB2FPYvNDSUiUl4kx8KOrwHYFHYbBxNSCfC2c2PbmiYHK5tUjJRRvRsFE1HFj1PpVnZemOa7/kNzQ4mIlBcbPgJXNtTqzNu7AwD3BIMK3naTg5VNKkbKKKvVwpjOtQH4V3wXDIsNDq+CuB3mBhMR8XSZabBxBgBxTcfz695TWCwwpoum8xaWipEy7ObImvg7bKxJ8CUhPGdmzbr3zQ0lIuLptuXMYKwYwftxjQC4pnEwEVU0nbewVIyUYYE+Xtzczn1/cvqF3Xy3fwmpp01MJSLiwVyu3JWv09vdyZebYgEY26WOmanKPBUjZdyYnIGsHxyuRka1FpCdDptmmxtKRMRTHfgFEvaCI4AvnL1IzXTSILgCXetXMTtZmaZipIyrW60CvRpVwzAsLPYf5m7cMA2cWabmEhHxSGveBcDV5g6mrXcvcja2a23tznuVVIx4gHFd3d2Dzx9shMuvGiQfh93fm5xKRMTDnIyGg8vBYmVVlZuIOZNGkK8XN7TR7rxXS8WIB+jRoCoNgiuQmGllS/Awd6P2qxERKVo5i5zReDDvbnL3Po/qWAs/h6bzXi0VIx7AYrEwoZu7d+S52E4YVjvErHEvVSwiIlfv3CnY9gUAB+qNYf3hM9itFkbnLLEgV0fFiIcY1qYGlf0dbEvy5USNAe5G9Y6IiBSNjTPAmQFhbfnvvsoADG5ZndAgH5ODeQYVIx7Cx8vG7R1rAfD2uT7uxh1fuat5EREpvOwM98QAILHVnXy/PQ6ACd3qmpnKo6gY8SC3d47AYbMyLzaU1KqtwJkJUbPMjiUiUrZt/wpS4yGwBtNOtyTbZdChdmVa1AwyO5nHUDHiQYIDfBjSOgyAL+yD3Y2a5isiUniGkTtwNbPdBD7ZcAKACd21yFlRUjHiYcbnTPN9JaYxTr9gOBcH0QtNTiUiUkYdWgknd4CXHwus/Ug6n0Wtyn70bRJidjKPomLEwzQNC6RLvSqku2z8Xmmou1H71YiIFE5Or4jRaiQfrHNvtTGua21sVi1yVpRUjHigC9N8nzneHsPqBcc2wLEok1OJiJQxCfth748ArA2+hYMJqQR427klMtzkYJ5HxYgH6t0omLrV/DmcXoH9wTm7+a7XNF8RkQK5sMhZwwG8tcUA4NYO4VTw1iJnRU3FiAeyWi1MzJlyNuVML3fjjvmQEmdeKBGRsiTtDGydC8CB+qNZe9C9yNmF7TekaKkY8VA3tq1BFX8Hy5LDOF25DbiyYONMs2OJiJQNUbMgKw1CmvPGvlAArm8VRlhFX3NzeSgVIx7Kx8vGmC61AZiWea27ceMM9+I9IiJyac4sWP8RAKdbTGTxDnev8p3dtchZcVEx4sFu7xSBj5eVjxKak+Eb4l60Z+c3ZscSESnddn4DKSfAP5ipp1vjMqB7g6o0DQs0O5nHUjHiwSr7O7ilXTjZ2Pnee6C7cd1U9yI+IiJyMcOAte8CkN56HJ9FnQTUK1LcVIx4uAnd6mCxwL/iOuGyecOJze6pviIicrGYte7/J23efObqS1qmk8ahAXRvUNXsZB5NxYiHq13VnwHNQjlDIBsDrnE3ahE0EZH85fSKOFvcwtSNyQDc1aMuFosWOStOKkbKgTt7uLsX/3Wqm7sheiEknzAxkYhIKXT2MOxeBMDSoJs5lZJBaKAPg1uGmZurHFAxUg60rVWJyIhKbHXW5mhAa3Blu2fWiIjI/6z7AAwXRt0+vLbZ/etxXNfaOOz6VVnc9BUuJ+7K6R15PbmPu2HjTMhKNzGRiEgpkp4Mmz4BYEuNkeyLP0cFbzu3daxlcrDyQcVIOdG3SQj1gyvwbUYbUrxDIS0BdnxtdiwRkdJh8yeQmQJVG/HSHvdtmVGdahHo42VysPJBxUg5YbVauLtHXZzYmJ3V19247n1N8xURcTlzB/YfbjCaqJhEHDYrE7T0e4lRMVKODG1dg+pBPkxL60621RvitkHMGrNjiYiYa/f3kBgDvpWZcrw1ADe1q0FwoI+5ucoRFSPliMNuZUK3OiQSwI/WHu5GTfMVkfJujXt33oQmo1iyNwmLBe7qUc/kUOWLipFy5rYOtQjy9eK/qTlrjuz6HhKPmhtKRMQsx6Pg6FqwevHf5N4ADGpenTpV/U0OVr6oGCln/L3tjOkcwR6jFtu8WoLhhI3TzY4lImKOnF6R1IZD+TTavZHoPT3VK1LSVIyUQ2O61MbHy8o7F3pHomZB1nlTM4mIlLik4xD9DQCfcB1Ol0G3+lVpUTPI3FzlkIqRcqhKBW9GRIbzs6sdp2whcP4sbPvC7FgiIiVrw0fgyiYrvAtv7vQF1CtiFhUj5dTE7nWxWG18kH5hmu8HmuYrIuVHZqp78Udgsf8NpGe5aFEjiK71q5gcrHwqVDHy3nvvUadOHXx8fGjXrh2rVq267PEZGRk8+eSTRERE4O3tTb169ZgxQ8uRmym8sh9DWoXxhbMnGRYfiN8Jh38zO5aISMnYOhfSE3FVrMOzu8MBuLdXPW2IZ5ICFyPz5s3jkUce4cknn2Tz5s10796dgQMHEhMTc8nXDB8+nF9++YXp06ezZ88e5s6dS+PGja8quFy9e3vVI5kKfJnV1d2gab4iUh64XLB2KgC/V72ZxHQX9aq5dzgXc1gMo2B98x07dqRt27ZMnTo1t61JkyYMGzaMKVOmXHT8jz/+yK233srBgwepXLlyoUImJycTFBREUlISgYGBhfoYkr+7P9nIgegofvb+O1is8NAWqBRhdiwRkeKzdwl8NhzDO5DuWe9yLM3Ga7e04qZ2Nc1O5nGu9Pd3gXpGMjMziYqKol+/fnna+/Xrx+rVq/N9zbfffktkZCT//ve/qVGjBg0bNuTxxx/n/HnN3igNHujdgP1GTVa5WoDhcg/oEhHxZGveBWBn6DCOpdmoWcmXIa3DTA5VvtkLcnBCQgJOp5OQkJA87SEhIcTFxeX7moMHD/Lbb7/h4+PDggULSEhI4L777uPMmTOXHDeSkZFBRkZG7vvJyckFiSkF0KJmED0aVmPm/v50d2yHTR9Dr8ng0II/IuKBTu6EQ79iWGw8Feu+RX1Pz3p42TSfw0yF+ur/eYCPYRiXHPTjcrmwWCzMmTOHDh06MGjQIF5//XVmzZp1yd6RKVOmEBQUlPsIDw8vTEy5Qg/0rs9yV2uOGCGQngTb5pkdSUSkeOQscnY0tC9bkgMIDvDmZt2eMV2BipGqVatis9ku6gWJj4+/qLfkgurVq1OjRg2Cgv63iEyTJk0wDINjx47l+5rJkyeTlJSU+zh6VMuVF6cOdSrTvnZVZmXn3H5bq918RcQDnYuH7e41lV4+2weAu3rUxcfLZmYqoYDFiMPhoF27dixdujRP+9KlS+nSpUu+r+natSsnTpzg3LlzuW179+7FarVSs2b+1ai3tzeBgYF5HlK87u9Tn6+cPUk1fCBhDxxcYXYkEZGitWE6ODM5U6kVixPDqeTnxciOtcxOJRTiNs2kSZOYNm0aM2bMYNeuXTz66KPExMRwzz33AO5ejdGjR+ceP3LkSKpUqcK4ceOIjo5m5cqV/O1vf2P8+PH4+voW3ZnIVenRoCq1a1TnS+eF3Xw/MDeQiEhRykqHDdMAmJrRH4DxXevg5yjQ0EkpJgUuRkaMGMGbb77JCy+8QOvWrVm5ciWLFy8mIsI9HTQ2NjbPmiMVKlRg6dKlJCYmEhkZyahRo7j++ut5++23i+4s5KpZLBYe6FOfj53uWzXG3h/hzEGTU4mIFJHtX0JaAuf9wphxpgUVvO2M7lzb7FSSo8DrjJhB64yUDJfLYNDbq/jH6afobdsKne6DARevHSMiUqYYBkztCvE7meY7npfO9uXBPvV5rF8js5N5vGJZZ0Q8m9Vq4aFrGjDLOQAAY9MnkJFicioRkat0cAXE7yTb7sfbZzvj77Axvmsds1PJH6gYkTwGNAslrmpnDriqY8lMga2fmx1JROTqrHVP511sv4Zk/BndpTaV/B0mh5I/UjEieVitFh64phGzc8aOONe+797HQUSkLDq1F/b9hIGFV5P64OewcWf3umankj9RMSIXGdSiOpsqDSDZ8MV2Zj8cWGZ2JBGRwlnn3kdtnaMjMUYId3SKoLJ6RUodFSNyEZvVwp19W/GlsxcA2WumXv4FIiKlUdoZ2DIXgDdSrsXHy8qdPdQrUhqpGJF8DW4ZxvLAobgMC/aDP0PCfrMjiYgUTNRMyD7PAXs91hmNub1jBFUreJudSvKhYkTyZbNauLFvd5a5WgOQpd4RESlLsjNhvXsX8nfS+uFtt3FXT/WKlFYqRuSShrQK4wf/oQAYWz5zb6InIlIWRH8DKbGcsVbme1dnRnasRXCAj9mp5BJUjMgl2W1WuvS9mX2uGjicaaRv+MTsSCIif80wYM27AEzL6IvV7uDenvVMDiWXo2JELmtomxp873s9AOm/T9U0XxEp/WLWQOwWMvDmM+c1jO4cQXCgekVKMxUjcll2m5UG104k2fCjYvoxUncuNjuSiMjl5fSKfJXdjQyvitytXpFST8WI/KWBbeuzxNu9CFrCL9rgUERKsTMHMXYvAmCGcwBjutTWDJoyQMWI/CWb1UKV3g/gNCxEJK4jOWa72ZFERPK37gMsGCx3tiLOqxZ3a12RMkHFiFyRXh0jWevoCMCBRW+YnEZEJB/pSRibPwVgunMQ47vV0R40ZYSKEbkiVqsFR5d7AWgU9z1nT8ebnEhE5E82fYwl8xx7XDXZ6mjNxG7qFSkrVIzIFYvsOYTDtgj8LBls+kZjR0SkFHFmY6x7H4DpzoFM7FaPID8vk0PJlVIxIlfMYrWS3nYiAI1iPic+MdXkRCIiOXZ/hyXpGAlGIMu9ejGuW22zE0kBqBiRAml07QRSLAHUtJxi6cKPzY4jIgKAK2c67xxnX8b3akKgj3pFyhIVI1IgFoc/yU1HAlD3wMfEnE4zOZGIlHvHNmI9toEMw85in+sY26W22YmkgFSMSIHVuPZBXFjpbI1m3vc/mB1HRMq57N/fAeBbZxduvyYSX4fN5ERSUCpGpOAqhpNcuz8ANfd/wt6TKSYHEpFyK/Eo1l3fArCowjBGtK9lciApDBUjUigVez8EwDDr70xdvN7kNCJSXmWsfh8rTn53NmNIv/447Pq1Vhbpqknh1OpMetVm+FoyCdn/BVuOJpqdSETKm4xzGFGzAPgx4EaGtq5hbh4pNBUjUjgWCz5d7wPgDvtSXvtxp8mBRKS8SVn3MT7OcxxwVafboJHYrBazI0khqRiRwmt+M06fytSwnMb/0E/8vj/B7EQiUl64XGT+7p7OuzRgGP2aVTc5kFwNFSNSeF4+2NqPB2Cc/Ude/mE3LpdhcigRKQ9ORS2kSsYxEg1/Wg6+D4tFvSJlmYoRuTrtJ2BYbHS07sZ1Yivfb481O5GIlANnl70JwKrAwXRpohk0ZZ2KEbk6gWFYmg4FYIxtCa8u2UNmtsvkUCLiyfZtXU3D81vIMmw0vH6S2XGkCKgYkavX8R4AhtpXc+5MHJ+tO2JyIBHxVIZhELvkNQC2BfWiUcPGJieSoqBiRK5eeAcIa4M3WdxqW8bby/aTkp5ldioR8UBrt0XTKXU5ADUHPm5yGikqKkbk6lksub0j4xy/kJyaxkcrD5ocSkQ8jctlcOiHt3BYnMRUaElIky5mR5IiomJEikazG8C/GtWM0wywbuCjVYeIT043O5WIeJDvNh2g//nFAFTu84i5YaRIqRiRomH3hkj3NN8H/H/mfJaTN3/ZZ3IoEfEUGdlOdv04jSqWFJJ9wqjQepjZkaQIqRiRohM5HqxeNM7aRQvLQeZtOMo+baInIkVg9u+HuDHTvSGeb7f7wKqdeT2JihEpOgGh7ts1wOQqv+J0Gfzf4l0mhxKRsu5MaiZRy7+mofU4WXZ/vCLHmB1JipiKESlaOQNZO6f9Sog1ieV7TvHbPi0TLyKF99bPexnp/B4Ae7vR4BNociIpaipGpGjVbAc1IrG4Mnk5IgqAlxZF49Qy8SJSCPvjz7F23Wp62rZhWKxYOt1jdiQpBipGpOh1uheAnsnfUtkHdsel8FXUUZNDiUhZ9PIPuxhjdc+gsTS+DirVNjeQFAsVI1L0mgyBCqFYU+N5rdlhAF79aS+pGdnm5hKRMmX1gQSidu3nRttv7oZO95sbSIqNihEpenYHtJ8AQM/E+URU8eNUSgYfaCE0EblCTpfBS9/vYpTtF3wsWRDWBmp1MjuWFBMVI1I82o0FmwPr8Y1M6ZABwIcrDxCbdN7cXCJSJszfdIz9sacZ47XU3dDpfvdqz+KRVIxI8agQDM1vBqBzwle0r12J9CwX//5xj8nBRKS0S83I5j9L9jDYuoZqJEJAGDQbZnYsKUYqRqT4dLwLAMvOBTzfuwoWCyzYfJxNMWdNDiYipdm7y/cTn5LOvT5L3A0d7gSbl7mhpFipGJHiE9YGwjuBK5umJ77mlnY1AXj+2524NNVXRPIRczqNaasO0cm6iwauQ+Dl577tKx5NxYgUr453u//dOIPHr6lNBW87W48l8fWmY+bmEpFS6V+Lo8l0uvhb4C/uhla3gV9lc0NJsVMxIsWryfXu+72ppwiO+YEH+9QH4JUf95CSnmVyOBEpTX7fn8CSnSepaz1J2/S17sacdYvEs6kYkeJl84IOE91vr53KuC61qVPVn4RzGby7/IC52USk1Mh2unj+u50ATKnxOxYMaNAfqjYwOZmUBBUjUvzajgWbN8RuwRG7gaeuawLAjN8OcTgh1dxsIlIqzFkXw96T56jlm0GHxB/cjZ3vMzeUlBgVI1L8/KtAy1vcb6/7gD6Ng+nZsBqZThcvLYo2N5uImO5saiavL90LwJsNtmLJSoWQ5lCnp8nJpKSoGJGSkbObL9ELsSSf4OnBTbFbLfy8K57le+LNzSYipnr1pz0knc+iWYgvbeK+dDd2uk+LnJUjKkakZIS2gIhuYDhh43TqB1dgXNfaADz37U7Ss5zm5hMRU2w7lshn62MAeLNlDJbk4+AfDC1uNjmZlCQVI1Jycqf5zoSs8zzctyEhgd4cOZ3GR9q3RqTccbkMnl64E8OAG1qH0eDgx+4n2k8Eu7e54aREqRiRktNoEASFw/kzsONrKnjb+ecg92DWd1fs5+iZNJMDikhJ+mLjUbYeTaSCt52nW5+D41Huwe6R482OJiVMxYiUHJvd/RcPwLr3wTAY0iqMTnUrk57l4sXvNZhVpLxITMvklR93A/DotQ2pvO0j9xMth0OFaiYmEzOoGJGS1XY02H0hbjscWY3FYuGFoc2xWy38FH1Sg1lFyon/LNnD2bQsGoUEMKaJBXZ9536ik6bzlkcqRqRk+VWGViPcb697H4CGIQEazCpSjvxx0OoLQ5th3/gRGC6o2xtCmpqcTsxQqGLkvffeo06dOvj4+NCuXTtWrVp1Ra/7/fffsdvttG7dujCfVjxFh5yBrLu/h8SjAHkGs36owawiHsv5x0GrbWrQsYYDNuUMXO18v7nhxDQFLkbmzZvHI488wpNPPsnmzZvp3r07AwcOJCYm5rKvS0pKYvTo0VxzzTWFDiseIqQp1Onh/ktowzQAKnjbefI6919E7yzfr5VZRTzUZ+uO5A5anTywMWz+FDKSoWpDqKffD+VVgYuR119/nQkTJjBx4kSaNGnCm2++SXh4OFOnTr3s6+6++25GjhxJ586dCx1WPMiFRdCiZsH5swBc37I63RtUJTPbxdMLd2AYhnn5RKTInUxO598/7gHg7wMaEVzBC9bm/O7odC9YNXKgvCrQlc/MzCQqKop+/frlae/Xrx+rV6++5OtmzpzJgQMHePbZZ6/o82RkZJCcnJznIR6m4QCo2gjSE+GnpwGwWCy8OLQ5DruVVfsS+HbrCXMzikiReuH7aFIysmlVM4hRHSNgz2JIPAK+laDlrWbHExMVqBhJSEjA6XQSEhKSpz0kJIS4uLh8X7Nv3z6eeOIJ5syZg91uv6LPM2XKFIKCgnIf4eHhBYkpZYHVBte/5X578ydwcAUAtav682Dv+gC8+H00SWlZJgUUkaK0fE88i7bFYrXAv25ogc1qgTXvuZ+MHA8OP3MDiqkK1Sdm+dN+AYZhXNQG4HQ6GTlyJM8//zwNGza84o8/efJkkpKSch9Hjx4tTEwp7SI6/2/dke8ehkz3omd39axL/eAKJJzL5JUlu00MKCJF4Xymk6e/2QHA+K51aF4jCE5shpjVYPWC9neanFDMVqBipGrVqthstot6QeLj4y/qLQFISUlh48aNPPDAA9jtdux2Oy+88AJbt27FbrezbNmyfD+Pt7c3gYGBeR7ioa55FgJrwtnDsPxfAHjbbfxrWHMAPlsXQ9SRMyYGFJGr9fayfRw7e56wIB8evTbnD9MLvSLNb4TA6uaFk1KhQMWIw+GgXbt2LF26NE/70qVL6dKly0XHBwYGsn37drZs2ZL7uOeee2jUqBFbtmyhY8eOV5deyj6fQBj8hvvtte+5l4MGOtatwi3tagLwz/k7yHK6zEooIldhT1xK7t5Tzw1phr+3HZJPwM757gO0yJlQiNs0kyZNYtq0acyYMYNdu3bx6KOPEhMTwz33uGdHTJ48mdGjR7s/uNVK8+bN8zyCg4Px8fGhefPm+Pv7F+3ZSNnUsB+0uMU91ffbh8DpHifyz0FNqOzvYM/JFK09IlIGOV0G//h6G9kug2ubhtCvWaj7ifUfgSsbIrpCWGtTM0rpUOBiZMSIEbz55pu88MILtG7dmpUrV7J48WIiIiIAiI2N/cs1R0QuMuBl8K0MJ3fAL88DUMnfwTOD3WuPvPXzPvbHnzMzoYgU0KzVh9lyNJEAbzsvDG3mbsxMg6iZ7rfVKyI5LEYZWMwhOTmZoKAgkpKSNH7Ek+34Gr7K2a2zz1PQ428YhsG4WRtYsecUkRGV+OLuzlitFw+WFpHS5eiZNPq9sZLzWU7+dUNz91RegA3TYdEkqFQbHtzknlknHutKf39rhRkpPZrfBNe+6H572Uuw5l0sFgv/uqEF/g4bG4+c5dN1R8zNKCJ/yTAMJs/fzvksJx3rVOa29rXcT7hc/1vkrOO9KkQkl4oRKV26PgS9n3S/veSfsGEaNSr68o+BjQF45YfdHE88b2JAEfkrX0Ud47f9CXjbrbx8U8v/9Wbu/xlO7wPvQGgzytyQUqqoGJHSp8ffoNuj7rcXPQab53B7xwgiIyqRmunkyQXbtVS8SCkVn5LOS4t2AfDotQ2pU/UPExXWvuv+t+1o8A4wIZ2UVipGpPSxWNzrj3S81/3+wvux7vyal29qicNmZcWeU3yz5bi5GUUkX899u5Ok81k0rxHIxG51/vfEyZ3ulZYtVuh4t2n5pHRSMSKlk8UCA6ZAu7GAAfPvov7p5TzctwEAz30bTXxyuqkRRSSvRdtiWbw9DpvVwis3tcRu+8OvmLU5i5w1GQIVa5kTUEotFSNSelkscN0b7g20DCd8OY67ww7SvEYgSeez+Kdu14iUGgnnMnh6oXvJ9/t71aNZWND/njx3CrZ96X678/0mpJPSTsWIlG5WKwx9F5oOA1cW9i/vYGqXVLxsFn7eFc+CzbpdI2I2wzB4+psdnEnNpHFoAA/0aZD3gI3TwZkBNSIhvIM5IaVUUzEipZ/NDjdNg0aDIDud8B/H8XJ796Z6z327k7gk3a4RMdN322L5YUccdquF14a3wmH/w6+WrHTYMM39dmctcib5UzEiZYPNC26eCfX6QFYaN+56hJtC4klOz2by/G26XSNikviUdJ7JuT3zYJ8GeW/PAPz+FqSecm+I2WSoCQmlLFAxImWHlw+MmAMR3bBkpPDv9GdpaT/K8j2n+DLqmNnpRModwzB4csEOEtOyaBYWyH296+U94Nf/wIr/c7/d4zF3L6dIPlSMSNni8IORn0PNDtgykpjn+zL1Lcd48btoLYYmUsIWbD7O0uiTeNnct2e8LsyeMQz45UVY/pL7/d5PQeR484JKqadiRMoe7wAY9SVUb4Vv1lm+8H2ZypnHeOyLLbhcul0jUhKOnU3j2YU7AXikb0Mah+bsO2IY8NNTsOpV9/v9XoKefzMppZQVKkakbPKtCHd8A8FNqew6w1zHvzh6cA/TfjtodjIRj+d0GUz6YispGdm0rVWRu3vUdT/hcsHix2HNO+73B70KXR40L6iUGSpGpOzyqwyjF0KVBoRZTvOZ4198smQtu2KTzU4m4tE+WnWQ9YfO4O+w8caI1u7FzbIzYf7EnJkzFrj+behwp9lRpYxQMSJlW4VgGPMtRqXaRFjjmWl7iWc/W0F6ltPsZCIeaeeJJF77aQ8Az17fjIgq/pBxDuaOgB1fgzVnKn67MSYnlbJExYiUfYFhWEZ/izOgBvWtJ3g+6Z+8s2i92alEPE56lpNH520hy2nQr2kIt0TWhNTTMPt6OLAMvPxh5DxocbPZUaWMUTEinqFSBLax35HhE0wT61H6bbqXtdGHzE4l4lFe+XE3e0+eo1qANy/f1BJL0lGY0R9ObALfyjDmO6jf1+yYUgapGBHPUaUe3uO/I9VWkZbWQ/h+OYIzZ8+YnUrEI/y69xQzfz8MwL9vbknl1AMwvT+c3gdB4TB+CdRsZ25IKbNUjIhnCW6Mdew3pOBPK2MPJz+4ASMzzexUImVafEo6j32xBYAxnSPo7XsIZgyAlBNQrbG7EKnW0NyQUqapGBGP4xvehlPD5nLO8KVJ+haOf3AzZGeYHUukTHK5DCbN20rCuUyaVA/kyQZH4eOhkJ4INTvAuB8gqIbZMaWMUzEiHqlu656sbP8eaYY3NU//TvInt4Mzy+xYImXO+ysP8Nv+BHy9bMxqsx/Hl6Mg+zw06OeeWu9X2eyI4gFUjIjHGnjdDbxX/SUyDC8Cj/xE9ld3gktTfkWu1KaYs7z2014APm+2npBlj4DhhJa3wq2fubdnECkCKkbEY1ksFibcMZYnvP5OpmHDvmsBLHzAvUqkiFxW0vksHpq7GafLxbTq39Jq92vuJzo/AMOmunfSFikiKkbEo1Xyd3DryAk8nP0g2YYVtn4Gix9z758hIvkyDIPJ87cRe/Yc7/lPp+/Zz91P9H3evdeMVb86pGjpO0o8Xse6VWjUexSTsu7FZVhg4wxY8qQKEpFLmLX6MMu2H+EjxxsMci4Diw2GvgvdHgGLxex44oFUjEi58GCfBpypO5R/ZOfslbH2XVj2krmhREqhTTFneWfxBj5xTKGPdRPYfeDWOdDmdrOjiQdTMSLlgs1q4a1bW7PKfwBPZ411N656FVb+x9RcIqXJ2dRMnvt0KXNsL9DeuhfDJ8i9O3ajgWZHEw+nYkTKjSoVvHlnZBs+M/rzr6yR7sZlL8Hqd8wNJlIKuFwGL3/6He9lTKax9SiuCqFYxv0AEZ3NjiblgIoRKVcia1dm8sDGfOQczJvOW9yNPz2Zs+25SPn15bff8vcTD1PTkkBmUB2sE36CkGZmx5JyQsWIlDsTutWhf7MQ3swaxie2G92Nix6DzXPMDSZiku0rv+G6zXdRxZLCmaCmOO5cCpUizI4l5YiKESl3LBYL/765FbWr+PN06k384D/M/cS3D8D2r0zNJlLSEtbMpfGy8VSwpLPPP5LK9/0EFaqZHUvKGRUjUi4F+XrxwR2R+Dns3Hv6FjZVGwaGC+bfBbu+MzueSInIXP0+lZfcixdOfvPuTvgD34F3gNmxpBxSMSLlVqPQAF67pRVg4aajNxMTPtS91PWX42DfUrPjiRQfw8BY9i8cP/0DKwZfWAZQ/555+PhqeXcxh4oRKdcGtqjO/b3rYWBl4OHhJNUdDK4smHc7HPzV7HgiRc/lhEWTsKz8NwBvZt9MndHvEVrJ3+RgUp6pGJFyb9K1jejVqBqpWRauPz6GzPoDIDsd5t4KMWvNjidSdLIz4KtxsHEGLsPCk1njqTL4GdrXqWJ2MinnVIxIuedeEK0Ntav4EZOUxdhz9+Gq2wey0uDTm+F4lNkRRa5eejJ8ehNELyQTO/dnPUR2m3Hc3rGW2clEVIyIgHtA60ejIwnwtrP68Dme9nkCo3Y3yEyBT26EuO1mRxQpvHPxMOs6OLyKVHwZk/kPTtbsz/NDm2HRXjNSCqgYEcnRICSA/45sg9UCczYlMKvWFKjZAdIT4eNhcGqP2RFFCu7MIZjeD+K2kWQNYnjGU8QERvLh6Eh8vGxmpxMBVIyI5NGrUTBPD24KwAtLj7I88j2o3grSEmD2EDh9wOSEIgUQtx1m9IezhzjjVZ2h55/hsFd9po+NpGoFb7PTieRSMSLyJ2O71GZUx1oYBtw//wB7rv0EgpvBuTj4eCgkxpgdUeSvHf4dZg6Ccyc5XaEB/VOeIobq/HdkGxqHBpqdTiQPFSMif2KxWHhuSDO61q9CWqaTcfP2c+qGz6FKA0g6CrOvh+QTZscUubTdi+CTGyAjmbNVI+l9+u+cohL/HNSEPo1DzE4nchEVIyL58LJZeW9kO+pW8+dEUjp3zDvMuVvnQ6XacPawu4fk3CmzY4pcbNMn7nVynBkk1rqWXicfItnw57YOtZjQrY7Z6UTypWJE5BKC/LyYPa4DVSt4szsuhXsWxpI5aiEE1oSEve6CJO2M2TFF3AwDVr3m3mPJcJHS5FauPTaRpCw7vRtV40XNnJFSTMWIyGWEV/Zj5tj2+Dls/LY/gSd+ScQYvRAqhEL8TndXeHqS2TGlvHO5YMk/4ZcXAEjr8BCDj4zgVJqTFjWCeGdkW+w2/XcvpZe+O0X+QouaQbw3qi02q4X5m4/zalQ2jF4IflUgdgvMuQUyzpkdU8qr7ExYcDesfQ+AzL7/4raD/Tly5jzhlX2ZMbY9/t52k0OKXJ6KEZEr0KtRMFNubAHAu8sP8MkBH7jjG/AJgqPr3EvHZ503N6SUP5mp8PltsP0LsNrJHvYBd+/rwNZjSVTKuc1YLUBTeKX0UzEicoWGR4bzaN+GADzz7U4WnqwCty8ARwAcXuUeNJidYXJKKTfSzrjXvtn/M3j54br1cx7b3Yjle07hbbcybUx76larYHZKkSuiYkSkAB66pj5jOkdgGPDYF1tZdq4mjPoSvPzcvxS+HAfOLLNjiqdLPOpezOz4RvCthDF6Ic9Eh7JwywnsVgvv396OdhGVzE4pcsVUjIgUgMVi4dnrm3FDmxpkuwzu/XQTa50N4ba5YPOGPYtg/l3ubdpFikP8bnchkrAXAmvA+CW8Gh3Ip2tjsFjg9RGt6d042OyUIgWiYkSkgKxWC/++uSV9mwSTke1i4uyNbHe0gRGfgtULds6HhQ+4ZziIFKWj62HmAEg+DlUbwoSf+CDazrvL3dsUvDSsOUNahZkcUqTgVIyIFIKXzco7I9vSqW5lzmVkM3rGOvYEdoabZ4DFBls/g8WPudd+ECkK+5a617Y5fxZqRML4JXy6y8mUH3YD8PcBjRjVMcLkkCKFo2JEpJB8vGxMG9OeVjWDOJuWxciP1rK3Sm+44QPAAhtnwJInVZDI1ds6L2fGVhrU7wtjvmXujnM89c0OAO7uWZf7etU3OaRI4akYEbkKFbztfDy+I81rBHI6NZORH61lf+gAGPJf9wFr34VlL5kbUsq2Ne/CgrvAlQ0thsNtn/PF1jNMnr8dgPFd6/DEgMYmhxS5OipGRK5SkJ8Xn07oSNPqgSScy+S2j9ZxIPwGGPSq+4BVr8LK/5gbUsoew4Cfn3OvrArQ6T644QO+2nKSf8zfBrh3mH56cBMt8y5lnooRkSJQ0c/BnIkdaRwawKmUDG77cC0H69wG/XJ6RZa9BKvfMTeklB3ObPceM7+94X7/mmeh//8xf8sJ/vbVVgwDRneO4Nnrm6oQEY9QqGLkvffeo06dOvj4+NCuXTtWrVp1yWPnz5/PtddeS7Vq1QgMDKRz584sWbKk0IFFSqtK/u6CpFFIAPEpGYz4cC176o6F3k+6D/jpSdgwzdSMUgZknYcv7oDNn4LF6r7l130Sn284ymNfuguR2zvV4vkh2vhOPEeBi5F58+bxyCOP8OSTT7J582a6d+/OwIEDiYmJyff4lStXcu2117J48WKioqLo3bs3119/PZs3b77q8CKlTZUK3sy58389JLd+uIbtde+Cbo+6D1j0GGyeY25IKb3OJ8InN8KexWD3cU8XbzuaGb8d4on523MLkReGNFchIh7FYhgFG+rfsWNH2rZty9SpU3PbmjRpwrBhw5gyZcoVfYxmzZoxYsQInnnmmSs6Pjk5maCgIJKSkggMDCxIXBFTJKZlMmbGerYeSyLA287MsZFE7v4PrJvq/mv3xo+gxc1mx5TSJDkWPr3JvRu0d5B7Ib3aXXl3+X7+s2QPAHf1qMvkgY1ViEiZcaW/vwvUM5KZmUlUVBT9+vXL096vXz9Wr159RR/D5XKRkpJC5cqVL3lMRkYGycnJeR4iZUlFPwefTuxIhzqVScnI5o4ZG/it3mPQbiwYLvcqrbu+MzumlBanD8CMfu5CpEIIjFuMEdGFV37cnVuIPNq3oQoR8VgF2lc6ISEBp9NJSEhInvaQkBDi4uKu6GO89tprpKamMnz48EseM2XKFJ5//vmCRAPA6XSSlaV9QQrKy8sLm81mdgyPE+Dj3jX17k+jWLn3FOM/3shbw//OwKx02Pa5ex+b2+ZCg2vNjipmOrEZPr0Z0hKgcl24YwHOoAie+WYHc9a5b38/OagJd/aoa3JQkeJToGLkgj9X5oZhXFG1PnfuXJ577jkWLlxIcPCl906YPHkykyZNyn0/OTmZ8PDwSx5vGAZxcXEkJib+dXjJV8WKFQkNDdVfXUXM12Hjo9HteGjuZpbsPMl9c7fw/HV/Y3TTdIj+xr3T78gvoG5Ps6OKGQ7+Cp+PhMxzUL0VjPqadO/KPPRpFD9Fn8RicS/xrpVVxdMVqBipWrUqNpvtol6Q+Pj4i3pL/mzevHlMmDCBL7/8kr59+172WG9vb7y9va8414VCJDg4GD8/P/1CLQDDMEhLSyM+Ph6A6tWrm5zI83jbbbw3qh3PLHT/pfvM93s42eNvPN4oA8ueH9wra96xAGp1MjuqlKSd38D8O8GZCXV6wIg5JLp8mDBtHVFHzuKwW3lrRGsGttDPpHi+AhUjDoeDdu3asXTpUm644Ybc9qVLlzJ06NBLvm7u3LmMHz+euXPnct111xU+bT6cTmduIVKlSpUi/djlha+vL+AuKoODg3XLphjYrBZeGtac0EAfXlu6l3dXHuFU60d5uW4G1oPL3N30YxZCjXZmR5WSsGG6e2YVBjQdCjd+xLEUJ2NmrObAqVQCfexMG9OeDnUuPbZOxJMUeGrvpEmTmDZtGjNmzGDXrl08+uijxMTEcM899wDuWyyjR4/OPX7u3LmMHj2a1157jU6dOhEXF0dcXBxJSUlFcgIXxoj4+fkVyccrry58/TTmpvhYLBYevKYB/765JTarhS+2JDAx/RGyw7tAZop7SmfcdrNjSnEyDFjxMiyaBBgQOR5unsm2uPPc+J67EKke5MNX93ZRISLlSoGLkREjRvDmm2/ywgsv0Lp1a1auXMnixYuJiHDf04yNjc2z5sgHH3xAdnY2999/P9WrV899PPzww0V3Flw8jkUKRl+/kjM8MpxpYyLxc9hYdvAcQ888SHpoO0hPhI+Hwak9ZkeU4uBywuLHYUXOEgg9/wHXvc53209yy/triE/JoFFIAPPv60LDkABzs4qUsAKvM2KGy81TTk9P59ChQ7krwkrh6OtY8qJPJDNx9gZOJKVTwyeDJZVfpcKZnVAhFMYthir1zI4oRSU7wz2dO/obwAKD/oMrciJv/bKPt37ZB0CfxsG8dWtrAny8TI0qUpSKZZ0RESk6TcMC+eaBrrSpVZHj6d70jHuYsxXqw7k4+HgoJOa/qrGUMRkpMOcWdyFi9YKbZ3C+9XgenLs5txC5s3sdPhodqUJEyi0VIybq1asXjzzyiNkxxETBAT7MvbMTN7SpwWlXBa5NeIx4Ry1IOgqzh0DyCbMjytU4dwpmDYZDv4KjAoz6kiPV+3Pj1NUs2h6Ll83Cv29qyZPXNcVm1a1SKb9UjJRihmGQnZ1tdgwpZj5eNl4f3oq/D2jEaUsQ1yf/nVhrKJw95O4hOXfK7IhSGGcPw4z+ELsF/KrCmO9YmtGUwf/9jV2xyVTxdzBnYieGt7/0Gkoi5YWKEZOMHTuWX3/9lbfeeguLxYLFYmHWrFlYLBaWLFlCZGQk3t7erFq1irFjxzJs2LA8r3/kkUfo1atX7vuGYfDvf/+bunXr4uvrS6tWrfjqq69K9qSk0CwWC/f1qs/H4zuQ5R/KLecnE0sVSNgLnwyDtDNmR5SCOLkTpveHMwcgqBbZY3/g39v9uPPjjaSkZ9O2VkUWPdRdM2ZEchRqBdbSzDAMzmc5Tfncvl62K56V8tZbb7F3716aN2/OCy+8AMDOnTsB+Pvf/86rr75K3bp1qVix4hV9vKeeeor58+czdepUGjRowMqVK7n99tupVq0aPXtqdc+yonuDanz/YDfum7OJ2479k3mOFwk5uQPXJzdiHbMQfILMjih/5cgamDsC0pMguCkJw+by0MI4Vh84DcC4rrWZPLAJDrv+FhS5wOOKkfNZTpo+s8SUzx39Qn/8HFf2JQ0KCsLhcODn50doaCgAu3fvBuCFF17g2muvfL+S1NRUXn/9dZYtW0bnzp0BqFu3Lr/99hsffPCBipEyJqyiL1/c3Zl/LQpi1Fp3QVIldjPps2/EZ+xC8K5gdkS5lN2L4atxkJ0OtTqzvO3bTJq2l7NpWfg5bLxyU0uubxVmdkqRUsfjihFPEBkZWaDjo6OjSU9Pv6iAyczMpE2bNkUZTUqIw27l+aHN+b5OZe6eD9ON5wiK3Uj8hzdQ7e6FWBxa5K/U2fwpfPsQGE6c9fvzou/fmPW5e7ZM0+qBvH1bG+oHq5AUyY/HFSO+XjaiX+hv2ucuCv7+/nnet1qt/Hk5mD+ulOpyuQBYtGgRNWrUyHNcQfb4kdJncMsw2tQazcsfe/PP008QfHo90W8OIeyeBVQM1MJYpYJhwO9vwc/PAnC24S0MP3Eb+xLc+z3d1aMuj/VriLdd2yyIXIrHFSMWi+WKb5WYzeFw4HT+9fiWatWqsWPHjjxtW7ZswcvLvSZB06ZN8fb2JiYmRrdkPFCNir689MAYvlnoz8At99M0bQMr3riBjBtm0r+lZmKYyuWCpU/DmncAWB92B7ftGIjTlU5wgDevD29NtwZVTQ4pUvppBJWJateuzbp16zh8+DAJCQm5PRx/1qdPHzZu3MjHH3/Mvn37ePbZZ/MUJwEBATz++OM8+uijzJ49mwMHDrB582beffddZs+eXVKnI8XIZrVw0w23cHzgDDLxopexgcwvJ/LAnA0knMswO1755MyCb+7JLUSmeo9n+MGBOF0wqEUoPz7SQ4WIyBVSMWKixx9/HJvNRtOmTalWrVqePX3+qH///jz99NP8/e9/p3379qSkpOTZjBDgxRdf5JlnnmHKlCk0adKE/v37891331GnTp2SOBUpIQ06DYYRn+K02LnetpZeu1+g32vL+Wbz8Ytu5UkxykyFubfBtnm4sDEp615eSepL1QreTB3VlvdGtaOyv8PslCJlhvamEUBfxzIn+luML8diMZx8mn0NT2WPp1PdKjw/pDmNQjWWpFilncH4bDiWYxtIx8E9mQ+zwtWGG9vW4JnBTanopyJE5ALtTSPiyZoOwXLDBxhYuN3+C8855rD24GkGvb2KF76LJjk9668/hhRc0nEyPuqP5dgGEg1/Rmb8kwMVuzBzbHteH95ahYhIIakYESmrWt6CZch/ARhrXcx7oYtxugxm/H6IPq+uYO76GLKd+Y9DkoI7e2QHie/2xvvsXmKNyoxyPkfPawaz9NGe9G4cbHY8kTJNxYhIWdb2Dhj0KgCDEuewvMNG6lbzJ+FcJpPnb6ffmyv5YXusxpNchdSMbD5fsABmDKBi5kkOuKrzVsQ7vD/pdh7u2wCfIprSL1KelY05sCJyaR3udK/4+dNT1Nn2Oj/1rczHDOad5fs5eCqVe+dsolV4Rf7RvxGd61W54i0LyruMbCfzNhxl/dIv+bfzP/hZMthra8Dpm+bwcotGZscT8SjqGRHxBF0ehN5PAmD/+SnGe//Cr3/rxUN96uPnsLH1aCIjp63j5vfXsGz3SfWUXEZaZjbTVh2kx7+Xs/G7j3jDOQU/SwbxwV2p/7fldFYhIlLk1DMi4il6/A2yzsNvr8Oixwiw+zKp3yju6Fybd5btY+6Go0QdOcv4WRtpUj2Q+3rVY1CL6tis6ikBSErLYvaaw8z8/RBn07IYa/uR5xwfA+BqdhPBN7wPdg1QFSkOmtorgL6OHsMw4MfJsG4qWKxw40fQ4mYA4pPTmf7bIT5de4TUTPfKvzUq+jKyYy1ubR9OlQrlc+uA6BPJfLL2MN9sPpGz47fBiwELuCPrK/cBHe+B/lPAqo5kkYK60qm9KkYE0NfRoxgGfP8oRM0Eiw2Gz4Ym1+c+nZiWyezVR5i12t0DAOCwWRnUIpQ7OkfQtlYljx9Xkp7lZMnOOD5Zc4SNR87mtjcN8WNqxU+JOJJTiPR5Gro/Bh7+9RApLipGpED0dfQwLhcsvA+2zgWrF9w2Fxrk3dU5PcvJ99ti+WTNYbYeS8ptj6jix7DWNRjWpgZ1qvr/+SOXWS6XwdpDp/lm83F+2BFHSno2AHarhf7NQxkTGUL7TX/DsnuRu1dp8JvQboy5oUXKOBUjkqt27do88sgjPPLII5c8Rl9HD+TMhvkTYecCsPvAyC+gbv4bKW49msgna4+waFtszq0Kt1Y1g7iuZXX6NA6hXjX/MtdjkuV0EXXkLD9Hn+T7bbHEJafnPhcW5MOI9rW4rUM4wY4MmDsSjvwGNm+4eXqe3iQRKZwrLUY0gFXEU9ns7jEj2RmwZzF8cgPUvwZajoBGg8Dhl3toq/CKtAqvyPNDmrE0+iQLNh/nt/0JbD2WxNZjSfzf4t1EVPGjT+Ng+jQOJjKiMr6O0rm+RnxKOqv3n+aX3fH8uiee5JweEIBAHzvXtazO0NY16FC7MlarBVLiYObNcHI7eAe6e5FqdzPxDETKHxUjZURmZiYOh0bySwHZvOCWWfD1BNj1Hez7yf1wBLj/8q9cN8/h/sAwYFg9SA3PZu/JFA7En+Po2fM4Ew1YC+vWwgYLhAT6ULOSL2EV3Q9/R8n/d+IyDM6ez2SXM5xFqU1YE5PK4dNpeY6p5OdF70bB9GsWSu/G1fC2/6GIOnPQXaSdPQz+wXD711C9ZcmehIioGDFLr169aN68OQCffvopNpuNe++9lxdffBGLxULt2rWZOHEi+/fvZ8GCBQwbNozZs2ezevVqnnjiCTZs2EDVqlW54YYbmDJlCv7+7nv78fHxTJgwgZ9//pnQ0FBeeuklM09TSgO7N4z4FE7the1fwLZ5kBgDWz+77Mv8gTY5D2w5jz9Ky3kcL4bMV8gKVAG6Aa0NH5a52vCDrSOx1brRpXE41zQJpnV4pfynL8duhU9vgtRTUKkO3DH/ouJMREqG5xUjhgFZaX99XHHw8ivQqPvZs2czYcIE1q1bx8aNG7nrrruIiIjgzjvvBOA///kPTz/9NE899RQA27dvp3///rz44otMnz6dU6dO8cADD/DAAw8wc+ZMAMaOHcvRo0dZtmwZDoeDhx56iPj4+KI/Vyl7qjWEPk9Br3/C0XWw+3vIPFfgD2MYkJqZzamUDOJTMjiVkuHemO9So88s4Otlw9fLhpfNipfd4v7XZr1o1UWn4R7nkeV0kel0kZXtIi3TSWb2pffY8bY66WGPpprrFENsaxhiWwNp06H2hxDROP8XHVoFc2+DzBQIbQGjvoaAkAJ/LUSkaHjeANbMVPi/MHOC/vMEOK5s9kGvXr2Ij49n586duYMCn3jiCb799luio6OpXbs2bdq0YcGCBbmvGT16NL6+vnzwwQe5bb/99hs9e/YkNTWVmJgYGjVqxNq1a+nYsSMAu3fvpkmTJrzxxhsawCrFJi0zm30nz7E7LpldsSnsjz/H8cTzHE88f9lCoiACfeyEVfQlvLIfjUMDaBwaSOPqAdSu4o/NAhzfBNHfuB+JMWC1w03TodmwvB8o+lv3bStnJkR0g9s+A5+gIskoInlpAGsZ0KlTpzyzEzp37sxrr72G0+mezRAZGZnn+KioKPbv38+cOXNy2wzDwOVycejQIfbu3Yvdbs/zusaNG1OxYsXiPREp9/wc9txBsH9kGAYJ5zI5nnieM6kZpKRnk5yeTfL5LM5lZOP6099CXlYrgb52Any8CPTxIsDHTkigD2EVfQjw8bp8iJrt3I9rnoUFd8OOr+Cr8eDMgpa3uI/ZOBMWTQLDBY0Hu4sVLxXfImbzvGLEy8/dQ2HW5y5CF8aBXOByubj77rt56KGHLjq2Vq1a7NmzB6DMTb8Uz2WxWKgW4E21gBJc3dVmhxs/dI+V2TIH5t/p7gVJPgHLc8ZQtR0Dg98Aa+mcESRS3nheMWKxXPGtErOtXbv2ovcbNGiAzZb/f5Bt27Zl586d1K9fP9/nmzRpQnZ2Nhs3bqRDhw4A7Nmzh8TExCLNLVLqWW0w5B33bKKoWe4F4C7o8Tf3poIq2kVKDW22YKKjR48yadIk9uzZw9y5c/nvf//Lww8/fMnj//GPf7BmzRruv/9+tmzZwr59+/j222958MEHAWjUqBEDBgzgzjvvZN26dURFRTFx4kR8fX1L6pRESg9rziqqHe7+X9vAf7sH8aoQESlVPK9npAwZPXo058+fp0OHDthsNh588EHuuuuuSx7fsmVLfv31V5588km6d++OYRjUq1ePESNG5B4zc+ZMJk6cSM+ePQkJCeGll17i6aefLonTESl9LBYY+AqEd4AKIVCnu9mJRCQfnjebpozo1asXrVu35s033zQ7ClB2v44iIlJ6XelsGt2mEREREVOpGBERERFTacyISVasWGF2BBERkVJBPSMiIiJiKhUjIiIiYiqPKUZcrqLZ/6K80tdPRETMUubHjDgcDqxWKydOnKBatWo4HA4th14AhmGQmZnJqVOnsFqtOBwOsyOJiEg5U+aLEavVSp06dYiNjeXECZP2pPEAfn5+1KpVC6vVYzrLRESkjCjzxQi4e0dq1apFdnZ27o63cuVsNht2u109SiIiYgqPKEbAvTuol5cXXl5/sc24iIiIlCrqkxcRERFTqRgRERERU6kYEREREVOViTEjFzYWTk5ONjmJiIiIXKkLv7cv/B6/lDJRjKSkpAAQHh5uchIREREpqJSUFIKCgi75vMX4q3KlFHC5XJw4cYKAgIAinX6anJxMeHg4R48eJTAwsMg+bmni6eeo8yv7PP0cPf38wPPPUedXeIZhkJKSQlhY2GXXsSoTPSNWq5WaNWsW28cPDAz0yG+wP/L0c9T5lX2efo6efn7g+eeo8yucy/WIXKABrCIiImIqFSMiIiJiqnJdjHh7e/Pss8/i7e1tdpRi4+nnqPMr+zz9HD39/MDzz1HnV/zKxABWERER8VzlumdEREREzKdiREREREylYkRERERMpWJERERETOXxxci//vUvunTpgp+fHxUrVryi1xiGwXPPPUdYWBi+vr706tWLnTt35jkmIyODBx98kKpVq+Lv78+QIUM4duxYMZzB5Z09e5Y77riDoKAggoKCuOOOO0hMTLzsaywWS76P//znP7nH9OrV66Lnb7311mI+m4sV5vzGjh17UfZOnTrlOaa0XD8o+DlmZWXxj3/8gxYtWuDv709YWBijR4/mxIkTeY4z6xq+99571KlTBx8fH9q1a8eqVasue/yvv/5Ku3bt8PHxoW7durz//vsXHfP111/TtGlTvL29adq0KQsWLCiu+FekIOc4f/58rr32WqpVq0ZgYCCdO3dmyZIleY6ZNWtWvj+T6enpxX0q+SrI+a1YsSLf7Lt3785zXGm6hgU5v/z+P7FYLDRr1iz3mNJ0/VauXMn1119PWFgYFouFb7755i9fUyp+Bg0P98wzzxivv/66MWnSJCMoKOiKXvPyyy8bAQEBxtdff21s377dGDFihFG9enUjOTk595h77rnHqFGjhrF06VJj06ZNRu/evY1WrVoZ2dnZxXQm+RswYIDRvHlzY/Xq1cbq1auN5s2bG4MHD77sa2JjY/M8ZsyYYVgsFuPAgQO5x/Ts2dO488478xyXmJhY3KdzkcKc35gxY4wBAwbkyX769Ok8x5SW62cYBT/HxMREo2/fvsa8efOM3bt3G2vWrDE6duxotGvXLs9xZlzDzz//3PDy8jI++ugjIzo62nj44YcNf39/48iRI/kef/DgQcPPz894+OGHjejoaOOjjz4yvLy8jK+++ir3mNWrVxs2m834v//7P2PXrl3G//3f/xl2u91Yu3ZtsZ7LpRT0HB9++GHjlVdeMdavX2/s3bvXmDx5suHl5WVs2rQp95iZM2cagYGBF/1smqGg57d8+XIDMPbs2ZMn+x9/lkrTNSzo+SUmJuY5r6NHjxqVK1c2nn322dxjStP1W7x4sfHkk08aX3/9tQEYCxYsuOzxpeVn0OOLkQtmzpx5RcWIy+UyQkNDjZdffjm3LT093QgKCjLef/99wzDc35xeXl7G559/nnvM8ePHDavVavz4449Fnv1SoqOjDSDPN8SaNWsMwNi9e/cVf5yhQ4caffr0ydPWs2dP4+GHHy6qqIVS2PMbM2aMMXTo0Es+X1qun2EU3TVcv369AeT5D9WMa9ihQwfjnnvuydPWuHFj44knnsj3+L///e9G48aN87TdfffdRqdOnXLfHz58uDFgwIA8x/Tv39+49dZbiyh1wRT0HPPTtGlT4/nnn899/0r/fyoJBT2/C8XI2bNnL/kxS9M1vNrrt2DBAsNisRiHDx/ObStN1++PrqQYKS0/gx5/m6agDh06RFxcHP369ctt8/b2pmfPnqxevRqAqKgosrKy8hwTFhZG8+bNc48pCWvWrCEoKIiOHTvmtnXq1ImgoKArznHy5EkWLVrEhAkTLnpuzpw5VK1alWbNmvH444/n7p5cUq7m/FasWEFwcDANGzbkzjvvJD4+Pve50nL9oGiuIUBSUhIWi+WiW5EleQ0zMzOJiorK83UF6Nev3yXPZc2aNRcd379/fzZu3EhWVtZljynpawWFO8c/c7lcpKSkULly5Tzt586dIyIigpo1azJ48GA2b95cZLmv1NWcX5s2bahevTrXXHMNy5cvz/NcabmGRXH9pk+fTt++fYmIiMjTXhquX2GUlp/BMrFRXkmKi4sDICQkJE97SEgIR44cyT3G4XBQqVKli4658PqSEBcXR3Bw8EXtwcHBV5xj9uzZBAQEcOONN+ZpHzVqFHXq1CE0NJQdO3YwefJktm7dytKlS4sk+5Uo7PkNHDiQW265hYiICA4dOsTTTz9Nnz59iIqKwtvbu9RcPyiaa5iens4TTzzByJEj82xyVdLXMCEhAafTme/PzqXOJS4uLt/js7OzSUhIoHr16pc8pqSvFRTuHP/stddeIzU1leHDh+e2NW7cmFmzZtGiRQuSk5N566236Nq1K1u3bqVBgwZFeg6XU5jzq169Oh9++CHt2rUjIyODTz75hGuuuYYVK1bQo0cP4NLXuaSv4dVev9jYWH744Qc+++yzPO2l5foVRmn5GSyTxchzzz3H888/f9ljNmzYQGRkZKE/h8ViyfO+YRgXtf3ZlRxzJa70/ODinAXNMWPGDEaNGoWPj0+e9jvvvDP37ebNm9OgQQMiIyPZtGkTbdu2vaKPfSnFfX4jRozIfbt58+ZERkYSERHBokWLLiq6CvJxC6KkrmFWVha33norLpeL9957L89zxXkNL6egPzv5Hf/n9sL8PBanwuaZO3cuzz33HAsXLsxThHbq1CnPIOuuXbvStm1b/vvf//L2228XXfArVJDza9SoEY0aNcp9v3Pnzhw9epRXX301txgp6McsboXNMmvWLCpWrMiwYcPytJe261dQpeFnsEwWIw888MBfzgqoXbt2oT52aGgo4K4Wq1evntseHx+fWxmGhoaSmZnJ2bNn8/x1HR8fT5cuXQr1ef/oSs9v27ZtnDx58qLnTp06dVEVm59Vq1axZ88e5s2b95fHtm3bFi8vL/bt23fVv8hK6vwuqF69OhEREezbtw8o/usHJXOOWVlZDB8+nEOHDrFs2bK/3Pq7KK9hfqpWrYrNZrvor6U//uz8WWhoaL7H2+12qlSpctljCvI9UFQKc44XzJs3jwkTJvDll1/St2/fyx5rtVpp37597vdsSbma8/ujTp068emnn+a+X1qu4dWcn2EYzJgxgzvuuAOHw3HZY826foVRan4Gi2z0SSlX0AGsr7zySm5bRkZGvgNY582bl3vMiRMnTBvAum7duty2tWvXXvHgxzFjxlw0A+NStm/fbgDGr7/+Wui8BXW153dBQkKC4e3tbcyePdswjNJz/Qyj8OeYmZlpDBs2zGjWrJkRHx9/RZ+rJK5hhw4djHvvvTdPW5MmTS47gLVJkyZ52u65556LBs8NHDgwzzEDBgwwdQBrQc7RMAzjs88+M3x8fP5yMOEFLpfLiIyMNMaNG3c1UQulMOf3ZzfddJPRu3fv3PdL0zUs7PldGKi7ffv2v/wcZl6/P+IKB7CWhp9Bjy9Gjhw5YmzevNl4/vnnjQoVKhibN282Nm/ebKSkpOQe06hRI2P+/Pm577/88stGUFCQMX/+fGP79u3Gbbfdlu/U3po1axo///yzsWnTJqNPnz6mTe1t2bKlsWbNGmPNmjVGixYtLpoW+ufzMwzDSEpKMvz8/IypU6de9DH3799vPP/888aGDRuMQ4cOGYsWLTIaN25stGnTptSfX0pKivHYY48Zq1evNg4dOmQsX77c6Ny5s1GjRo1Sef0Mo+DnmJWVZQwZMsSoWbOmsWXLljxTCTMyMgzDMO8aXpg2OX36dCM6Otp45JFHDH9//9yZB0888YRxxx135B5/YVrho48+akRHRxvTp0+/aFrh77//bthsNuPll182du3aZbz88sulYmrvlZ7jZ599ZtjtduPdd9+95DTr5557zvjxxx+NAwcOGJs3bzbGjRtn2O32PEVqaT2/N954w1iwYIGxd+9eY8eOHcYTTzxhAMbXX3+de0xpuoYFPb8Lbr/9dqNjx475fszSdP1SUlJyf88Bxuuvv25s3rw5d6Zdaf0Z9PhiZMyYMQZw0WP58uW5xwDGzJkzc993uVzGs88+a4SGhhre3t5Gjx49LqqGz58/bzzwwANG5cqVDV9fX2Pw4MFGTExMCZ3V/5w+fdoYNWqUERAQYAQEBBijRo26aIrdn8/PMAzjgw8+MHx9ffNddyImJsbo0aOHUblyZcPhcBj16tUzHnrooYvW6igJBT2/tLQ0o1+/fka1atUMLy8vo1atWsaYMWMuujal5foZRsHP8dChQ/l+T//x+9rMa/juu+8aERERhsPhMNq2bZunJ2bMmDFGz5498xy/YsUKo02bNobD4TBq166db4H85ZdfGo0aNTK8vLyMxo0b5/lFZ4aCnGPPnj3zvVZjxozJPeaRRx4xatWqZTgcDqNatWpGv379jNWrV5fgGeVVkPN75ZVXjHr16hk+Pj5GpUqVjG7duhmLFi266GOWpmtY0O/RxMREw9fX1/jwww/z/Xil6fpd6MG51Pdbaf0ZtBhGzkgVERERERNonRERERExlYoRERERMZWKERERETGVihERERExlYoRERERMZWKERERETGVihERERExlYoRERERMZWKERERETGVihERERExlYoRERERMZWKERERETHV/wMuoFpKRBnt9QAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "net = MLP(key=jr.PRNGKey(0))\n", - "optim = Optim(net)\n", - "\n", - "\n", - "@jax.jit\n", - "def train_step(net, optim, input, target):\n", - " grads = jax.grad(loss_func)(net, input, target)\n", - "\n", - " # argnums=1 -> return the updated optim state\n", - " @ft.partial(sp.value_and_tree, argnums=1)\n", - " def apply_optim(grads, optim):\n", - " return optim(grads)\n", - "\n", - " grads, optim = apply_optim(grads, optim)\n", - " net = jax.tree_map(lambda p, g: p + g, net, grads)\n", - " return net, optim\n", - "\n", - "\n", - "for i in range(1, 10_000 + 1):\n", - " net, optim = train_step(net, optim, input, target)\n", - " if i % 1_000 == 0:\n", - " loss = loss_func(net, input, target)\n", - " print(f\"Epoch={i:003d}\\tLoss: {loss:.3e}\")\n", - "\n", - "plt.plot(input, target, label=\"true\")\n", - "plt.plot(input, net(input), label=\"pred\")\n", - "plt.legend()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Line search\n", - "\n", - "In this section [backtracking line search](https://en.wikipedia.org/wiki/Backtracking_line_search) is implemented. The line search is used to find the step size that satisfies the strong [Wolfe conditions](https://en.wikipedia.org/wiki/Wolfe_conditions). for more check [N&W Ch3](https://www.math.uci.edu/~qnie/Publications/NumericalOptimization.pdf). The method is written in a stateful manner, i.e. it modifies the state of the optimizer inplace, However it is executed in a functional manner using `at` method to comply with `jax` transformations." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import jax\n", - "import sepes as sp\n", - "import jax.numpy as jnp\n", - "from typing import Callable\n", - "import jax.tree_util as jtu\n", - "import functools as ft\n", - "\n", - "# transform numpy function that work on array to\n", - "# work on pytree of arrays. additionally if the rhs is a scalar it will\n", - "# be broadcasted to the pytree\n", - "tree_mul = sp.bcmap(jnp.multiply)\n", - "tree_add = sp.bcmap(jnp.add)\n", - "tree_neg = sp.bcmap(jnp.negative)\n", - "tree_vdot = sp.bcmap(ft.partial(jnp.vdot, precision=jax.lax.Precision.HIGHEST))\n", - "\n", - "\n", - "def mask_field(**kwargs):\n", - " return sp.field(\n", - " # un mask when the value is accessed\n", - " on_getattr=[lambda x: sp.tree_unmask(x, cond=lambda node: True)],\n", - " # mask when the value is set\n", - " on_setattr=[lambda x: sp.tree_mask(x, cond=lambda node: True)],\n", - " **kwargs,\n", - " )\n", - "\n", - "class BackTrackLS(sp.TreeClass):\n", - " \"\"\"Backtracking line search with strong Wolfe conditions.\n", - "\n", - " Args:\n", - " func: the function to be optimized with respect to the loss function.\n", - " accepts single pytree argument. for multiple arguments use\n", - " `functools.partial` to fix the other arguments.\n", - " maxiter: the maximum number of iterations.\n", - " tol: the tolerance for the stopping criterion.\n", - " c1: the sufficient decrease parameter for the Armijo condition. Must\n", - " statisfy 0fields for more details\n", - " func: Callable = mask_field()\n", - "\n", - " def __init__(\n", - " self,\n", - " *,\n", - " func: Callable[..., jax.Array],\n", - " maxiter: int = 100,\n", - " tol: float = 0.0,\n", - " c1: float = 1e-4,\n", - " c2: float = 0.9,\n", - " step_size: float = 1.0,\n", - " decay: float = 0.5,\n", - " ):\n", - " self.func = func\n", - " self.maxiter = maxiter\n", - " self.tol = tol\n", - " # wolfe conditions\n", - " self.c1 = c1 # armijo condition constant\n", - " self.c2 = c2 # curvature condition constant\n", - " self.step_size = step_size\n", - " self.decay = decay\n", - "\n", - " # conditions numerics\n", - " self.wolfe1 = jnp.inf\n", - " self.wolfe2 = jnp.inf\n", - " self.error = jnp.inf\n", - "\n", - " # status\n", - " self.tol_reached = False\n", - " self.max_iter_reached = False\n", - " self.fail = False\n", - " self.iter_count = 0\n", - "\n", - " def step(self, xk0, fk0: jax.Array, dfk0):\n", - " \"\"\"Compute the next iterate of the line search.\n", - "\n", - " Args:\n", - " xk0: the initial parameters. accepts pytree.\n", - " fk0: the initial function value.\n", - " dfk0: the initial gradient. accepts pytree same structure as xk0.\n", - "\n", - " Returns:\n", - " xk1: the next iterate. xk1 = xk0 + αpk\n", - " fk1: the next function value. f(xk0 + αpk)\n", - " dfk1: the next gradient. ∇f(xk0 + αpk)\n", - " \"\"\"\n", - " # NOTE: calling this method will raise `AttributeError` because\n", - " # its mutating the state (e.g. self.something=something)\n", - " # it will only work if used with `.at` that executes it functionally\n", - "\n", - " self.step_size = jnp.minimum(1.0, self.step_size)\n", - " # for simplicity we will use the negative gradient as the descent direction\n", - " pk = tree_neg(dfk0)\n", - " # <∇f(xk), pk> but with pytrees\n", - " dfkTpk0 = sum(jtu.tree_leaves(tree_vdot(dfk0, pk)))\n", - " # xk+1 = xk + αpk\n", - " xk1 = tree_add(xk0, tree_mul(pk, self.step_size))\n", - " # f(xk+1), ∇f(xk+1)\n", - " fk1, dfk1 = jax.value_and_grad(self.func)(xk1)\n", - " # <∇f(xk+1), pk> but with pytrees\n", - " dfkTp1 = sum(jtu.tree_leaves(tree_vdot(dfk1, pk)))\n", - "\n", - " # armijo condition\n", - " # f(xk+1) ≤ f(xk) + c1α∇f(xk)⊤pk\n", - " self.wolfe1 = fk1 - (fk0 + self.c1 * self.step_size * dfkTpk0)\n", - "\n", - " # curvature condition\n", - " # |∇f(xk+1)⊤pk| ≤ c2|∇f(xk)⊤pk|\n", - " self.wolfe2 = abs(dfkTp1) - self.c2 * abs(dfkTpk0)\n", - " self.error = jnp.maximum(self.wolfe1, self.wolfe2)\n", - " self.error = jnp.maximum(self.error, 0.0)\n", - " self.iter_count += 1\n", - "\n", - " # check status\n", - " self.tol_reached = self.error <= self.tol\n", - " self.max_iter_reached = self.iter_count >= self.maxiter\n", - " self.fail = self.fail | (self.max_iter_reached & ~self.tol_reached)\n", - " self.step_size = jnp.where(\n", - " self.fail | self.tol_reached,\n", - " self.step_size,\n", - " self.step_size * self.decay,\n", - " )\n", - " return xk1, fk1, dfk1\n", - "\n", - " @staticmethod\n", - " def cond_func(state) -> bool:\n", - " *_, ls = state\n", - " return ~(ls.fail | ls.tol_reached | ls.max_iter_reached)\n", - "\n", - " @staticmethod\n", - " def body_func(state):\n", - " (xk0, fk0, dfk0), _, ls = state\n", - " # NOTE: calling this method will raise `AttributeError` because\n", - " # its mutating the state (e.g. self.something=something)\n", - " # it will only work if used with `value_and_tree` that executes it functionally\n", - " (xk1, fk1, dfk1), ls = sp.value_and_tree(lambda ls: ls.step(xk0, fk0, dfk0))(ls)\n", - " return (xk0, fk0, dfk0), (xk1, fk1, dfk1), ls" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "ls = BackTrackLS(\n", - " func=ft.partial(loss_func, input=input, target=target),\n", - " maxiter=100,\n", - " tol=1e-4,\n", - " c1=1e-4,\n", - " c2=0.9,\n", - " step_size=1.0,\n", - " decay=0.9,\n", - ")\n", - "\n", - "# example usage\n", - "# pass the initial parameters, function value and gradient\n", - "fk0, dfk0 = jax.value_and_grad(loss_func)(net, input, target)\n", - "init = (net, fk0, dfk0)\n", - "state = init, init, ls\n", - "state = jax.lax.while_loop(ls.cond_func, ls.body_func, state)\n", - "_, (xk1, fk1, dfk1), ls = state" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "dev-jax", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -}