diff --git a/docs/API/recurrent.rst b/docs/API/recurrent.rst index 2fe915d..b6e7e64 100644 --- a/docs/API/recurrent.rst +++ b/docs/API/recurrent.rst @@ -3,8 +3,6 @@ Recurrent .. currentmodule:: serket.nn -.. autoclass:: RNNCell - .. autoclass:: LSTMCell .. autoclass:: GRUCell .. autoclass:: SimpleRNNCell @@ -24,7 +22,4 @@ Recurrent .. autoclass:: FFTConvGRU2DCell .. autoclass:: FFTConvGRU3DCell -.. autoclass:: ScanRNN - - -.. autofunction:: scan_rnn \ No newline at end of file +.. autofunction:: scan_cell \ No newline at end of file diff --git a/docs/notebooks/train_bilstm.ipynb b/docs/notebooks/train_bilstm.ipynb index 8dacf2e..f9fc894 100644 --- a/docs/notebooks/train_bilstm.ipynb +++ b/docs/notebooks/train_bilstm.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -37,10 +37,12 @@ "import optax # for gradient optimization\n", "import serket as sk\n", "import functools as ft\n", + "from typing_extensions import Annotated\n", "import time\n", "\n", "EPOCHS = 100\n", - "LR = 1e-3" + "LR = 1e-3\n", + "Input = Annotated[jax.Array, \"Float[seq_len, input_dim]\"]" ] }, { @@ -52,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -91,26 +93,32 @@ " key: jax.Array,\n", " ):\n", " k1, k2, k3 = jr.split(key, 3)\n", - " self.rnn1 = sk.nn.ScanRNN(\n", - " cell=sk.nn.LSTMCell(in_features, hidden_dim, key=k1),\n", - " backward_cell=sk.nn.LSTMCell(in_features, hidden_dim, key=k2),\n", - " return_sequences=True, # return all outputs of the sequence\n", - " )\n", - " self.rnn2 = sk.nn.ScanRNN(\n", - " # in_features is hidden_dim*2 (for each cell from previous layer)\n", - " cell=sk.nn.LSTMCell(hidden_dim * 2, out_features, key=k3)\n", - " )\n", + " self.cell1 = sk.nn.LSTMCell(in_features, hidden_dim, key=k1)\n", + " self.cell2 = sk.nn.LSTMCell(in_features, hidden_dim, key=k2)\n", + " self.cell3 = sk.nn.LSTMCell(hidden_dim * 2, out_features, key=k3)\n", "\n", - " def __call__(self, x):\n", - " return self.rnn2(self.rnn1(x))\n", + " def __call__(self, input: Input) -> Input:\n", + " # initialize the states of the cells\n", + " state = sk.tree_state(self)\n", + " # run the forward cell\n", + " output1, state1 = sk.nn.scan_cell(self.cell1)(input, state.cell1)\n", + " # run the backward cell\n", + " output2, state2 = sk.nn.scan_cell(self.cell2, reverse=True)(input, state.cell2)\n", + " # concatenate the outputs\n", + " output = jnp.concatenate((output1, output2), axis=1)\n", + " # run the final cell\n", + " output, state3 = sk.nn.scan_cell(self.cell3)(output, state.cell3)\n", + " # return the last time step\n", + " return output[-1]\n", "\n", "\n", - "nn = BiLstm(1, 64, 1, key=jax.random.PRNGKey(0))\n", + "key = jax.random.PRNGKey(0)\n", + "net = BiLstm(1, 64, 1, key=key)\n", "# 1) mask the non-jaxtype parameters\n", - "nn = sk.tree_mask(nn)\n", + "net = sk.tree_mask(net)\n", "# 2) initialize the optimizer state\n", "optim = optax.adam(LR)\n", - "optim_state = optim.init(nn)" + "optim_state = optim.init(net)" ] }, { @@ -122,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -132,26 +140,26 @@ "\n", "\n", "@ft.partial(jax.grad, has_aux=True)\n", - "def loss_func(nn, x, y):\n", + "def loss_func(net: BiLstm, x: jax.Array, y: jax.Array):\n", " # pass non-jaxtype over jax transformation\n", " # using `tree_mask`/`tree_unmask` scheme\n", " # 3) unmask the non-jaxtype parameters to be used in the computation\n", - " nn = sk.tree_unmask(nn)\n", + " net = sk.tree_unmask(net)\n", " # 4) vectorize the computation over the batch dimension\n", " # and get the logits\n", " # here we dont vectorize over state argument so we use `None`\n", - " logits = jax.vmap(nn)(x)\n", + " logits = jax.vmap(net)(x)\n", " # 5) use the appropriate loss function\n", " loss = mse(logits, y)\n", " return loss, (loss, logits)\n", "\n", "\n", "@jax.jit\n", - "def train_step(nn, optim_state, x, y):\n", - " grads, (loss, logits) = loss_func(nn, x, y)\n", + "def train_step(net: BiLstm, optim_state: optax.OptState, x: jax.Array, y: jax.Array):\n", + " grads, (loss, logits) = loss_func(net, x, y)\n", " updates, optim_state = optim.update(grads, optim_state)\n", - " nn = optax.apply_updates(nn, updates)\n", - " return nn, optim_state, (loss, logits)" + " net = optax.apply_updates(net, updates)\n", + " return net, optim_state, (loss, logits)" ] }, { @@ -163,29 +171,29 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 100/100\tBatch: 100/100\tBatch loss: 1.632614e-03\tTime: 0.019\r" + "Epoch: 100/100\tBatch: 100/100\tBatch loss: 2.065103e-03\tTime: 0.022\r" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAACIYklEQVR4nOzdd3hT5dvA8W9G996LsvfepQxBQNkCIiKgiMoQxYUTBw5+7i3iAhkqiqIMZe+9C2VToC3QPaB0z+S8f+RNoHTQQpOu+3NduWzPec7JnViau8+4H5WiKApCCCGEENWEurIDEEIIIYQoD0lehBBCCFGtSPIihBBCiGpFkhchhBBCVCuSvAghhBCiWpHkRQghhBDViiQvQgghhKhWJHkRQgghRLWirewAKpperyc2NhYnJydUKlVlhyOEEEKIMlAUhfT0dPz9/VGrS+9bqXHJS2xsLIGBgZUdhhBCCCFuQ1RUFHXq1Cm1TY1LXpycnADDi3d2dq7kaIQQQghRFmlpaQQGBpo+x0tT45IX41CRs7OzJC9CCCFENVOWKR8yYVcIIYQQ1YokL0IIIYSoViR5EUIIIUS1UuPmvAghhKgZdDod+fn5lR2GqEBWVlZoNJo7vo8kL0IIIaqcjIwMoqOjURSlskMRFUilUlGnTh0cHR3v6D6SvAghhKhSdDod0dHR2Nvb4+XlJQVHawhFUUhKSiI6OpomTZrcUQ+MJC9CCCGqlPz8fBRFwcvLCzs7u8oOR1QgLy8vLl68SH5+/h0lLzJhVwghRJUkPS41T0X9P5XkRQghhBDViiQvQgghhKhWJHkRQgghzKhPnz48//zzlR1GjSLJixBCCFFFbN++HZVKxbVr1yo7lCpNkhdRqyiKwk8//cTx48crOxQhhBC3SZIXUav8+eefTJ06lXbt2lV2KEKIcsrMzCzxkZOTU+a22dnZZWp7uzFOmDABR0dH/Pz8+Pzzzwud//XXX+ncuTNOTk74+voybtw4EhMTAbh48SJ33303AG5ubqhUKiZOnAjA+vXr6dmzJ66urnh4eDB06FDCw8NvK8aaQJIXUavUq1fP9PXt/nISQlQOR0fHEh+jRo0q1Nbb27vEtoMGDSrUtn79+sW2ux0vv/wyO3bsYNWqVWzcuJHt27dz5MgR0/n8/Hxmz57NsWPHWLlyJRcvXjQlKIGBgfzzzz8AhIWFERcXx9dffw0Yfl/NmDGDw4cPs2XLFtRqNSNHjkSv199WnNWdFKkTtUpwcDABAQHExMRw5MgRevXqVdkhCSFqiIyMDH7++Wd+++03+vXrB8DixYupU6eOqc3jjz9u+rphw4Z88803dOnShYyMDBwdHXF3dwcMyZerq6up7c3J2YIFC/Dy8uL06dO0bt3ajK+qapLkRdQ6nTt3JiYmhsOHD0vyIkQ1kpGRUeK5m6u1GodiiqNWFx50uHjx4h3FZRQeHk5eXh5BQUGmY+7u7jRr1sz0fUhICO+88w7Hjh0jJSXF1HNy+fJlWrZsWeK9z58/z6xZszhw4ADJycmFrpPkRYga7q+//jJ9fejQoUqMRAhRXg4ODpXe9k5kZmYyYMAABgwYwJIlS/Dy8uLy5csMGDCAvLy8Uq8dNmwY9erVY968efj7+6PX62nduvUtr6upJHkRNUZmZiZ2dnZF/qoySk1NZcyYMabvJXkRQlSkRo0aYWVlxYEDB6hbty4AKSkpnDt3jt69e3P27FmuXLnCRx99RGBgIACHDx8udA9ra2vAsDml0ZUrVwgLC2PevHmm3uLdu3db4iVVWTJhV1RLiqIU+YujefPm2NjY8O233xZ7zblz5wCwsrICICoqivT0dPMGKoSoNRwdHXniiSd4+eWX2bp1KydPnmTixImmP6jq1q2LtbU1c+bMISIign///ZfZs2cXuke9evVQqVSsXr2apKQkMjIycHNzw8PDg59++okLFy6wdetWZsyYURkvscqQ5EVUSxcuXMDV1ZVBgwahKApgGA8vKChg0aJFxV5jTF6Cg4MJDQ0lLS0NJycnS4UshKgFPv30U3r16sWwYcPo378/PXv2pFOnToBhR+VFixaxbNkyWrZsyUcffcRnn31W6PqAgADeffddXnvtNXx8fJg+fTpqtZqlS5cSEhJC69ateeGFF/j0008r4+VVGSrF+Ju/hkhLS8PFxYXU1FScnZ0rOxxhJj///DOTJk2iZ8+e7Nq1C4CzZ8/SokULrK2tycjIMPWwGM2aNYvZs2czefJkfvrpp8oIWwhRBjk5OURGRtKgQQNsbW0rOxxRgUr7f1uez2/peRHV0s6dOwG46667TMeaNWuGi4sLeXl5nD59usg1YWFhADRt2tQyQQohhDALSV5EtVRc8qJSqWjfvj0AR48eLXKNcdioWbNmJCQkMHnyZAYOHGj+YIUQQlQoSV5EtRMVFcXFixfRaDR079690LkOHToARZMXvV5fKHmxt7fn559/ZsOGDcTHx1smcCGEEBVCkhdR7RjnuHTs2LHIhFtj8nLzsJGiKCxbtowvvviCBg0a4OTkRIsWLQBZMi2EENWNJC+i2iluyMho2LBhhIWFsWHDhkLHNRoNgwcP5oUXXjBN5O3SpQsgyYsQQlQ3kryIaicoKIjBgwdzzz33FDnn5uZG06ZNSyxUdyNj8nJzkSghhBBVm1TYFdXOY489xmOPPVauazZs2EBKSgo9evQwVba8sedFURRUKlWFxyqEEKLiSc+LqHE2btzI2LFj+eqrr0zHvv76a8aOHcv69etNx9q1a4eVlRXJyclcunSpEiIVQghxOyR5EdXKkSNHiImJKbXNxYsXWbp0KevWrTMdK67Gi42NDW3btqVx48ay4kgIUa3Ur1+/0B9oKpWKlStX3tE9K+IeliLDRqJamTRpEkePHmXlypUMHz682DY3Lpc27oFk3PL+xq3pAfbs2YONjY1ZYxZCCHOLi4vDzc2tTG3feecdVq5cSWho6G3fo7JJ8iKqjdTUVNM/tq5du5bYrk2bNmg0GpKSkoiJiSEtLQ29Xo+zszM+Pj6F2kriIoSoLHl5eaZdpO+Ur69vlbiHpciwkaiSNmzYUGh+Chh6SRRFoUmTJvj5+ZV4ra2tramGy9GjRwsNGZU0KVev11PDtvkSQlhYnz59mD59OtOnT8fFxQVPT0/eeust0++W+vXrM3v2bCZMmICzszNTpkwBYPfu3fTq1Qs7OzsCAwN59tlnyczMNN03MTGRYcOGYWdnR4MGDViyZEmR5755yCc6OpqxY8fi7u6Og4MDnTt35sCBAyxatIh3332XY8eOoVKpUKlUps1sb77HiRMn6Nu3L3Z2dnh4eDBlyhQyMjJM5ydOnMiIESP47LPP8PPzw8PDg6effpr8/PwKfFeLJ8mLqHKSk5MZOnQogwYN4t133zX9w9+xYwdQfH2Xm3Xs2BEwzJG5sbJucYYOHYqrq6upnRCialEUhay8gkp5lPePmsWLF6PVajl48CBff/01X3zxBfPnzzed/+yzz2jXrh1Hjx7lrbfeIjw8nIEDBzJq1CiOHz/On3/+ye7du5k+fbrpmokTJxIVFcW2bdv4+++/+e6770hMTCwxhoyMDHr37k1MTAz//vsvx44d45VXXkGv1zNmzBhefPFFWrVqRVxcHHFxcYwZM6bIPTIzMxkwYABubm4cOnSIZcuWsXnz5kJxAWzbto3w8HC2bdvG4sWLWbRokSkZMicZNhJVzr59+ygoKAAMY7PJycnMmTOn1OJ0N+vQoQO//PILR48exd3dHSh5Q8Zr166Rnp7O6tWrS0xwhBCVJztfR8tZG27d0AxOvzcAe+uyf1QGBgby5ZdfolKpaNasGSdOnODLL79k8uTJAPTt25cXX3zR1H7SpEmMHz+e559/HoAmTZrwzTff0Lt3b77//nsuX77MunXrOHjwoKm8w88//2zqXS7O77//TlJSEocOHTL9/mvcuLHpvKOjI1qtttRhot9//52cnBx++eUXHBwcAPj2228ZNmwYH3/8sWkI3s3NjW+//RaNRkPz5s0ZMmQIW7ZsMb1ec5GeF1Hl7N27FzAkGxqNhp49e5KZmWkqJlfW5EWr1ZKfn8+sWbNYtWoVo0ePLrbtxIkTAfjkk08KddUKIUR5devWrdDwdHBwMOfPn0en0wHQuXPnQu2PHTvGokWLcHR0ND0GDBiAXq8nMjKSM2fOoNVq6dSpk+ma5s2b4+rqWmIMoaGhdOjQwZS43I4zZ87Qrl07U+IC0KNHD/R6vWkoHqBVq1ZoNBrT935+fqX2ClUUs/a87Ny5k08//ZSQkBDi4uJYsWIFI0aMKPWa7du3M2PGDE6dOkVgYCBvvvmm6cNF1A7G5OXVV1+lT58+NGzYkC1btlBQUEBgYCD16tW75T169OhBRkaGaUJu/fr1S2z76KOP8sEHHxAZGcn333/PSy+9VCGvQwhRMeysNJx+b0ClPXdFujEZAMMQz9SpU3n22WeLtK1bt+5tDWfb2dnddnzlZdxuxUilUqHX683+vGbtecnMzKRdu3bMnTu3TO0jIyMZMmQId999N6GhoTz//PNMmjSpyD41oubS6XScPHkSgO7du9OwYUPA0JPyxx9/8O6775apEq5Wqy3zSiIrKytmzZoFwMcff1xoQpoQovKpVCrsrbWV8ihv5e0DBw4U+n7//v00adKkUO/EjTp27Mjp06dp3LhxkYe1tTXNmzenoKCAkJAQ0zVhYWFcu3atxBjatm1LaGgoV69eLfa8tbW1qSeoJC1atODYsWOFeqP37NmDWq2uEsPrZk1eBg0axP/+9z9GjhxZpvY//PADDRo04PPPP6dFixZMnz6dBx54gC+//NKcYYoqRKPREBcXx8GDBwvNUXF3d+ehhx4q97YA58+f5/333y9UsK44Dz/8MI0bNyY5OZlvv/32tmIXQojLly8zY8YMwsLC+OOPP5gzZw7PPfdcie1fffVV9u7dy/Tp0wkNDeX8+fOsWrXKNDG2WbNmDBw4kKlTp3LgwAFCQkKYNGlSqb0rY8eOxdfXlxEjRrBnzx4iIiL4559/2LdvH2DoiY6MjCQ0NJTk5GRyc3OL3GP8+PHY2try6KOPcvLkSbZt28YzzzzDI488UqTkRGWoUnNe9u3bR//+/QsdGzBggOkNL05ubi5paWmFHqJ6s7a2pkuXLmXaXLE069ato2nTprz55pt89tlnpbbVarW8/fbbACxcuNAi3Z5CiJpnwoQJZGdn07VrV55++mmee+4505Lo4rRt25YdO3Zw7tw5evXqRYcOHZg1axb+/v6mNgsXLsTf35/evXtz//33M2XKFLy9vUu8p7W1NRs3bsTb25vBgwfTpk0bPvroI1Pvz6hRoxg4cCB33303Xl5e/PHHH0XuYW9vz4YNG7h69SpdunThgQceoF+/flXmj7sqtdooPj6+SEbn4+NDWloa2dnZxWaaH374Ie+++66lQhTVyI3dvWXp5hw7dizXrl3j0UcfvePESQhRO1lZWfHVV1/x/fffFzlnrPR9sy5durBx48YS7+nr68vq1asLHXvkkUcKfX/zku569erx999/F3s/GxubYs/dfI82bdqwdevWEuMqbkn0jVsWmFO1/w09c+ZMUlNTTY+oqKjKDkncgf79+zNlyhQSEhLu+F7GbQIAAgICbtleo9Ewffp0nJyc7vi5hRBCmE+VSl58fX2LfGglJCTg7Oxc4viejY0Nzs7OhR6ieoqKimLLli0sWLAAR0fHO77fjb14gYGB5bpWURTOnDlzxzEIIYSoeFUqeQkODmbLli2Fjm3atIng4OBKikhYknGJdPv27YssJ7xdq1at4qWXXmL8+PFlviY+Pp4uXbrQqlUrPvjggxIrbG7btq3QCgAhRO22fft2iw2b1HZmTV4yMjIIDQ01baZnnN18+fJlwDDkM2HCBFP7J598koiICF555RXOnj3Ld999x19//cULL7xgzjBFFWFMXrp3715h97zvvvv49NNPS1ymWBxvb2+6du2Koii88cYbPPDAA6SnpxdpN2/ePLp3717iOLYQQgjzMGvycvjwYTp06GCaezBjxgzTTGowbL9tTGQAGjRowJo1a9i0aRPt2rXj888/Z/78+QwYUDnFicTtWbt2bbEf9rdijuTldqjVar777jt++uknrK2tWb58OUFBQWzfvp2YmBhTO41GQ15eHu+9914lRiuEELWPSqlhW+mmpaXh4uJCamqqzH+pBN988w3PPfccQ4cOZeXKlWXu8cjMzMTFxQWdTselS5eoW7eumSMtm/379zNq1ChiY2MBGDNmDEuXLjWdCw4ORq1Wc+bMmRL3TjKXlJQUHBwcsLa2tujzCmFuOTk5REZGUr9+fYtWixXml52dzcWLF2nQoAG2traFzpXn87tKzXkR1V9QUBC2trasXr2aV155pczXHT58GJ1OR0BAQLkn15pTt27dCAkJoWfPngDExMSQk5NjOjd06FD0er2pRoylJCUlERgYSN++fS36vEJYgvGPnry8vEqORFQ04//T8gzlF0d6XkSFyMjI4IcffqB///6EhYXx0EMPAfDjjz+WWqDJaM2aNcyYMYN27drx119/mTvcctPpdISHh9OkSZNC9WOMG6CBYYO1tm3bWiSepUuXMnbsWMDw3lfUBGchqgJFUbh8+TL5+fn4+/tL3aUaQq/XExsbi5WVFXXr1i2y9UJ5Pr8leREVYu3atQwZMoQGDRoQERHB7NmzmTVrFlqtlvXr19OvX78y3aegoACttkrVTrylMWPG8Ndff3HfffexatUqizzn4sWLTRuWHjx4kC5duljkeYWwlLy8PCIjI6XadQ2jVqtp0KBBscPd5fn8rl6fEqLK2rx5M4Bpe4c333yTsLAwlixZwqhRo1i/fj3dunW75X2qW+IC8O677/L3338TGhpKSkoKbm5uZn/O6Oho09fHjx+X5EXUONbW1jRp0kSGjmoYa2vrCulJq36fFKJKMtbnMfawqFQq5s+fT2RkJHv37uXUqVMlJi+5ubloNJpqmbgANG/enPXr13PXXXeVeSfrOzV9+nS2bNnCtm3bOHHihEWeUwhLU6vVRSZ1CgEybCQqQEJCAr6+vgAkJibi5eVlOpeens7SpUuZPHlysdfqdDq++uor3nnnHSZPnswXX3xhkZjNSVGUImO55hAeHk5mZibNmzeXFUdCiGpPho2ERRk37mrfvn2hxAXAycmpUOKSnJzM448/Tvfu3dm1axe7d+827QReEz6A9Xo9I0eO5J577uHpp582axLTqFEjs91bCCGqMklexB27eb5LaZ577jn+++8//vvvP9MxZ2dn+vTpw1NPPWW2GC3l77//5t9//+Xff/9lz549zJs3r0L2abrZq6++ioeHB1OnTsXFxaXC7y+EEFWZJC/iju3ZswcoOXnR6xXScwu4lpXHozPeJtU+gMzsHNq0akXbtm1o0rgR1lot6VotMdeycbWzwt5aY5Ghl4o2evRoYmNjefnll1m6dCnHjh3jn3/+oUWLFhX2HNnZ2XzyyScA2NnZERoaypQpUwgKCqqw5xBCiKpM5ryIIjIzM3nppZcIDAzk1VdfvWUxoZycHPbu3Uu3bt1QW9lwOi6NE9GpHI9O5UTMNcKTMtHpy/djplWr8HC0xs/FDn9X2///rx31Pexp4OlAoLs9VpqqW/thz549PPjgg8TGxuLs7MyxY8eoX79+hdz7woULNGnSBHt7ewYPHszff//NZ599xosvvlgh9xdCiMogc17EHYmKiuKHH37AwcGByZMnF5nHcjNbW1u8mnbinbXn+fdYLDn5xddlsLPS4GZvhYu9NdZaNXq9gk6voFcU8nV60nIMvTP5OoUCvUJCWi4JabmERhW9l0atoq67PY28HGnh50RzX2da+DlRz8MBjbrye2x69OjB0aNHGTJkCIcPH+bJJ59k3bp1FdKbZFwmXadOHdq0acPff//N8ePH7/i+QghRXUjyIoowfjhmZmYyZswY04Tcm+Xk6/jvWCy/7b/EsehU03FPR2va1nGldYALbQNcaOHvjIeDNbZWty4HrSgK2fk6UrLySU7PJS41m9hrOcSlZhNzLZuLyVlEJmeSna8jMjmTyORMNp9JMF1vZ6WhdYAz7QNdaR/oRvu6rvi72FbKEJS3tze//fYb7dq1Y8eOHZw9e7ZCho+iogzZXGBgoKmiryyXFkLUJpK8iCKMH44AR44cKXbp76nYVJ5ecoSLV7IAsNKoGNLGj4e71aNTPbfbThZUKhX21lrsrbUEuNrRLtC1SBtFMfTKRCRlcC4hnbPx6ZyJSyMsIZ3sfB2HLqZw6GIKEAmAt5MNQQ09CGrgTreGHjTycrBYMtOsWTMWLVpE586dady4cYXc8+aeF4DTp09Xy+rEQghxO+Q3nSjixuqtqamphIeHmz54FUXhtwOXmb36NHkFegrSr5B3ciMn/v0JHxd7i8SnUqnwdbHF18WW7o09Tcd1eoXI5AxCo1IJjUohNOoaZ+PSSUzP5b9jsfx3zLAztKejDT0ae9C7qRe9mnjh5WTewnLGfZ4qyo3JS4MGDXBwcCAzM5Pz589X6MRgIYSoqiR5EUXcmLyAofelcePGpOXkM/OfE6w5EQdAPW0auxdMZ9iAvhZLXEqjUato7O1EY28nHuhUB4DsPB3Hoq+xP+IK+yOucOTyNZIzclkVGsuqUEMy0zrAmT5Nvbm3lQ9tAlzM2iuzZ88e8vPz6dOnz23f48bkRa1W07p1aw4cOMCJEyckeRFC1AqSvIgijMNGVlZW5OfnExISQrd+Q3lkwQEuXclCq1bx2qDm/PHuFPQ56WWq71JZ7Kw1dGvoQbeGHgDkFug4evkau84nseNcEidj0kyPb7ddwN/Flntb+TKglS9d6ruhrcAVTcuXL2fUqFEEBgZy6tQpnJycbus+CxcuJCoqylTVuG3bthw4cIBLly5VWKxCCFGVyVJpUUTbtm05ceIEo0ePZtmyZdx9z0C0g17jXEIGAa52fDuuA43dtHh4eJCfn8/Zs2dp1qxZZYd9W5LSc9l1PonNZxLYHpZEVp7OdM7T0YZh7fwY0T6AtnXuvEcmMzOTNm3aEBkZyauvvspHH310p+EDhi0Z7OzsbjsZEkKIqqA8n9+SvIgiUlNTiY6O5urVq9x1V2/8H5yFVYMueDvZ8N8zPfFxtmXFihXcf//9NGrUiPPnz1fLgnI3y8nXsft8MutPxbP5TALXsvJN5xp6OjC8fQAPdK5DgKvdbT/H33//zejRo6lfvz4RERE14n0TQoiKIHVexB1xcXHBxcWFvLw8gh5/h3ivTlhpVPzwSCd8nA07vK5ZswaAIUOG1JgPYFsrDf1b+tC/pQ/5Oj27ziex4mgsm07HE5GcyZebz/HVlnP0aerF2K516dvcu9zDSoMGDcLGxoaLFy/e1tLpuLg4PvzwQxo1asRzzz1XrmuFEKKmqLolSkWl23b+KvFenQB4f2QbOtZ1M53z8fHB39+fIUOGVFZ4ZmWlUdO3uQ9zxnbg8Jv38MWD7Qhu6IGiwLawJKb8GkL3j7byxaZzJKbnlPm+Dg4Opsm6a9euLXdc58+fZ86cOXz77beFjr/zzjv07t2bw4cPl/ueQghR3UjyIgo5duwY06ZN44O5C5nxZygAE7vX58HOgYXavf/++0RHR1fpyboVxdFGy/0d6/DHlG5se6kPU3s3xMPBmsT0XL7Zcp6eH23jpWXHOBOXVqb7DR48GLi95OXGlUY3OnDgADt37iQkJKTc9xRCiOpGkhdRSGhoKD/8NJ9F57Vk5ukIbujBY+1d2LdvX5G2KpUKtbp2/Qg18HRg5qAW7JvZjzljO9Cxrit5Oj1/h0Qz6OtdjJ+/n93nkyltKpkxeYmIiCA/P7/EdsUpKXkxFquTSrtCiNpA5ryIQqKjo3FqP5A8W3c8HW14sq2W+vUCcXd3Jzk5GZVKxYkTJ2jZsuUtN2ysyay1aoa182dYO3+OXE7h592RrD8Zz54LV9hz4Qod6rrybL8m9GnqVWROUOPGjTlx4gStWrUq93yhkpIX4zYBsseREKI2kORFFBIZHY9Lj7EAvHBPE7q198XKyoqrV69y6dIl7O3tadeuHV5eXly4cEGW5wId67rRcZwbMdeymbczgj8OXubo5Ws8tvAQbeu48GzfJvRr4V0oUWnduvVtPVdZel6K285BCCFqktrV5y9uKTTPG429C55W+YzpHIiNjY3pgzEkJIR169ahKAoBAQGSuNwkwNWOd+5rxa5X72ZyrwbYWWk4Hp3KpF8O8+CP+wi5lFLkGr1eX+oQ082MBQRvTl6aN2+OVqvl2rVrRSokCyFETSPJizCJuZZNsrshURnd1Nq0DLhTJ8OKo5CQENMS6aFDh1ZOkNWAt5Mtbwxpye5X72Zq74bYaNUcupjCqO/38uSvIYQnZQDw5JNP4uPjU655KiX1vNjY2JgKBcq8FyFETSfJizD5fEMYaKzIuXScwe2vry7q2LEjAPv372fDhg0ANXaJdEXycLRh5qAW7Hj5bsZ0DkStgvWn4rn3y5288+8pLsclkZycXK5VRydPniQ0NJSWLVsWOde2bVv8/PzIy8uryJchhBBVjlTYFQCcjEll2JzdKEDcoudIOHsYV1dXAA4dOkTXrl1Nbb28vIiPj691K43u1LmEdD5Zf5bNZxIBsFfruPzfN3R0z2fnjh2F2mZlZWFnZ1euuStpaWk4OTnJfBchRLVUns9v+fQRKIrCB2vPoACDW3pxZMsqXFxcTOfbtGmDVnt9bvegQYMkcbkNTX2cmP9oF5ZMCqKRlwNZeg2eQ17gfN372H/2+jyVBQsW4OHhweOPP16u+zs7O0viIoSoFeQTSLDjXBJ7w69grVEzc2jrIkt4bW1t+eCDD0zfy5DRnenR2JN1z93FzEHNoSAXm4DmjF0UyodrzzDjldd44oknyMnJYcmSJaSkXJ/ku3v3bp555hn++OOPUu+v1+vJyMgw98sQQohKI8mL4Pvt4QBMCK5HoLt9sW1eeukldu/ezcyZM7n33nstGV6NZK1VM7V3I4ZwmMwzO1FQ8ePOCDbSEZuA5jg7O5Ofn8+KFStM1xw4cIBvv/2W1atXl3jf+fPn4+vry1tvvWWJlyGEEJVCkpda7mRMKgcir6JVq/BJPc20adOK/XBUqVT06NGDDz74wDQXRty5UYP7k/zvJ+RtmYO3kw0Zagf8Hv6MXs99jcrKhqVLl5ralrRM+kYuLi4kJSWxadMms8cuhBCVRZKXWmjjxo1s2bIFgJ93RwIwpK0fR/ds5YcffuDgwYOVGV6t0rNnT4KDg5l4T0dWPdmF0Z3qoAAn87zwe/Rrdp28SGKiYYJvScukb9S3b19UKhWnTp0iNjbWEi9BCCEsTpKXWubSpUsMGDCA/v37czoylv+OGT7gnujZoEwfjqJiWVtbs3fvXj744AP8PFz4dHQ7Fj3WBV9nW6w86uD7yOf8eigOvV4p0/8fDw8PU12ezZs3W+Q1CCGEpUnyUsv8+uuvpq8/WbGfAr1C1/rutK3jWqZhCWF+fZp5s/75Xgxo5YOiUjNnVzSPLjxIdLJh1+pb/f+55557AGToSAhRY0nyUosoisKiRYsAUFnZcOCKNQCP92wAlG1YQliGq701PzzcifdHtsbWSs2u88moBr2Bbb12ZU5eNm/eXK6tB4QQorqQ5KUW2bNnD+Hh4djb2zNv03GydSrquttzT0sfcnJySE5OBiR5qSpUKhUPdvTn1Q4qfGzy0Ti44v3ge6wMyyw1KenevTt2dnbEx8dz8uRJC0YshBCWIclLLWLsdXlwzBiWHDLMdXmsR300ahUxMTEA2Nvb4+bmVlkhipv89NNPPD5qEOotXzC0lRcqtYaP14cx/fejZOYWFHuNjY0NU6ZM4bXXXpMq00KIGkl76yaiphg9ejQpKSl0HPII2w5l4mijoZu34S94Y/JSp04dqdJahYwcOZJnnnmG/Xt2sXSJD0GNvXn3v9OsORHHuYR0fnykEw29HItc99VXX1k+WCGEsBDpealFBgwYwD///MO+q3YAJOxdztNTJwFw1113ce3aNdPGi6Jq8Pf3p3fv3gAsW7aMR4Lrs3RKN7ydbDifmMGIuXvYF36lkqMUQgjLkuSlljkTl8be8CuoVZByYCW7d+82lZJ3cXGhfv36lRugKGLMmDEAvPzyy2zdupXO9d1Z/UxPOtR1JS2ngAkLDvBPSHSR6zIyMlizZg3nzp2zdMhCCGFWkrzUApcvX+bNN9/k/PnzLN57EYCBrX2p5+VEfn4+27Ztq9wARalGjRpl+nrVqlUAeDvb8sfkbgxp40e+TuHFZcf4ctO5QhN5p02bxtChQ1m8eLHFYxZCCHOS5KUW+OWXX3j//feZ9PRzrDhqmNvyWI8GDBgwAIANGzbwv//9jyeffJIjR45UZqiiGF5eXvTo0QOABx54wHTc1krDnLEdeLJ3IwC+3nKeGX8dI7dAB0C/fv0AqfcihKh5JHmp4W6s7dJk4ERyC/S08nemcz03U/Kyfv16VqxYwY8//igl5auo9evXc+TIEXr16lXouFqt4rVBzfnw/jZo1CpWHI1h8i8hZOUV0L9/fwAOHz5MampqZYQthBBmIclLDWes7eLo5MzJHHcAJnavj0qlom/fvmi1WsLDw009LoGBgZUZriiBo6MjHTp0KPH82K51WTCxC3ZWGnaeS2LCzwdx8vChYcOGKIrC/v37LRitEEKYlyQvNdyyZcsAuGvcM8Sn5eLhYM2wdv4AODk5mYYjjKRAXfXVu6kXv00KwtlWy+FLKYz9aT9d7zIMHe3evbuSoxNCiIojyUsNd+HCBQDSfDsCMC6oLrZWGtP5adOmMW3aNABsbW1xd3e3fJCiwnSq58bSKcF4OlpzOi6NsDpD0Dh5SvIihKhRJHmp4S5fvoyVd0OicmzQqlU83K1eofNjxozhoYceAgxDRlKgrvpr6e/MX1OD8Xex5Wq+Ft/xH3PodDj5+fmVHZoQQlQISV5quLi4OJw7DwNgcBs/fJxti7SR3aRrnoZejvw9rTv1PezRuvjQ6pl5JGZI8iKEqBkkeanhTl24hFu7ewGY2KN+sW0yMzMBsLa2tlRYwgL8Xe1YOiWYeh72JGQUMH7+AeJTcyo7LCGEuGOSvNRwf4bEkq9XaFfHhQ6BrsW2CQ4Opnv37rzxxhuWDU6Yna+LLb9P7kYdNzsuXsli3Lz9JKZLAiOEqN4keanBMnILWLAnEoDHezYocT5LmzZt2LNnT5EaIqJmcLXSM8TuPLa6LCKSMxk37wDJGbmVHZYQQtw2SV5qsDcWrONaVj4e1jqGtPGr7HBEJdFoNLzz8rOEz38WT3stFxIzmLjwIBm5BZUdmhBC3BZJXqqh7Oxs4uLiSm2TlVfA+kuGMvEBqSfQauR/dW1la2tLly5dKLgWz0M+CXg4WHMyJo2pvx42bSUghBDViXyiVWFffvkl06ZN48MPP+T3339n06ZNPPfcc/j7+/P888+Xeu3vBy6TixX5KXF09ZH/zbVdz549ATh7aCcLH+uCvbWGPReu8OJfx9DrlVtcLYQQVYu2sgMQJfv333/Zvn17sedCQ0M5f/48u3fvxs3NjREjRpjO5eTr+HFnBABp+5fR4K4JFohWVGU9e/bk448/Zvfu3cyv48qPj3Ti8UWHWH08Dk9HG94e1lJq/Aghqg35k7wKe+qpp3jrrbeYMGECffr0oUmTJowcOZJ169Zx5swZ9u/fz+OPP84333xT6LqlBy+TlJ4LmVfJOLlV9isSdO/eHYCwsDCSkpLo1cSLz0a3A2DR3ot8tz28MsMTQohykZ6XKmz06NGMHj26xPNNmjQB4Ny5c6ZjOfk6vt9h+CBK3b8M9AXUrVvXvIGKKs/d3Z1WrVpx6tQp9u7dy/DhwxnePoArGXm8t/o0n24Io46bHcPbB1R2qEIIcUvS81IFxcbG8v777/Pnn3+W2s6YvMTExJgKzS0LiSYhLRcvRyuuHV2HSqUiIEA+kAT06NEDtVpNePj1XpbHezZgcq8GALz893GOXk6prPCEEKLMJHmpgk6dOsWbb77J7NmzS23n4eFh2kjxwoUL5Bbo+H6bYSPGEU3tUSt6/Pz8sLKyMnvMoup79913uXbtGjNmzCh0/LVBLejfwpu8Aj2Tfwkh9lp2JUUohBBlY5HkZe7cudSvXx9bW1uCgoI4ePBgiW0XLVqESqUq9LC1LbofT00WExMDUKYeE2Pvy/nz55m3M4LY1By8nWx4cWQwubm5HDlyxKyxiurD19cXJyenIsc1ahVfPdSB5r5OJGfk8sTiw2RKDRghRBVm9uTlzz//ZMaMGbz99tscOXKEdu3aMWDAABITE0u8xtnZmbi4ONPj0qVL5g6zSrmd5CUk7CJzthp6Xd4Y0gJbKw1arRYfHx/zBSpqDEcbLfMf7YynozVn4tJ44c9QWUIthKiyzJ68fPHFF0yePJnHHnuMli1b8sMPP2Bvb8+CBQtKvEalUuHr62t61LYP4PIkL02bNgVgY7IruQV6ght6cF87f7PGJ6qvJUuW0L17dz777LMi5+q42fPThM5Ya9VsPJ3A55vCKiFCIYS4NbMmL3l5eYSEhNC/f//rT6hW079/f/bt21fidRkZGdSrV4/AwECGDx/OqVOnzBlmlVOe5OWhhx7ikyXruWLjh5VGxewRrVCpVLz99tuMHz+ePXv2mDtcUY3Ex8ezb98+Dhw4UOz5jnXd+PSBtgDM3RbOxlPxlgxPCCHKxKzJS3JyMjqdrkjPiY+PD/Hxxf9SbNasGQsWLGDVqlX89ttv6PV6unfvTnR0dLHtc3NzSUtLK/So7mJjY4GyJS8B9Rqw8rJhQu7kXg1p7G2Y07BhwwZ+//33UofnRO3Tpk0bAE6cOFFim+HtA3i8h2EF0ot/HSMyOdMisQkhRFlVudVGwcHBTJgwgfbt29O7d2+WL1+Ol5cXP/74Y7HtP/zwQ1xcXEyPmlCQrTw9L3O2XiDmWjYBrnY807eJ6fjly5cBasT7ISqOMXk5f/482dklryqaObg5Xeq7kZ5bwLTfQsjKkwm8Qoiqw6zJi6enJxqNhoSEhELHExIS8PX1LdM9rKys6NChAxcuXCj2/MyZM0lNTTU9oqKi7jjuyrZ582Y2bNhAs2bNSm13PiGdef+/DUCT9FBiLkcChuE6Y8+WFKgTN/L19cXDwwO9Xs+ZM2dKbGelUTN3XEc8HW04G5/OGytOoigygVcIUTWYNXmxtramU6dObNmyxXRMr9ezZcsWgoODy3QPnU7HiRMn8PPzK/a8jY0Nzs7OhR7VXcuWLbn33ntxcHAosU1egZ6X/z5OgV7B7up5Fn/womkJekxMDIqiYGNjg5eXl6XCFtWASqUq09ARgLezLXPHdUCjVrHiaAy/7a9dq/6EEFWX2YeNZsyYwbx581i8eDFnzpxh2rRpZGZm8thjjwEwYcIEZs6caWr/3nvvsXHjRiIiIjhy5AgPP/wwly5dYtKkSeYOtVr5dMNZQqOu4WSrpSOGiqnGbQKMvU+BgYGy2Z4ooqzJC0BQQw9eG9gcgPdWn+ZY1DVzhiaEEGVi9r2NxowZQ1JSErNmzSI+Pp727duzfv160yTey5cvo1Zfz6FSUlKYPHky8fHxuLm50alTJ/bu3UvLli3NHWqVEBoaypo1a2jfvj1Dhgwpts2m0wnM22UYIvr0gXYcWxPK7xjmMYDMdxGla9OmDYGBgdjb25ep/aReDThyOYV1J+N5dulR1jzbC0cb2RZNCFF5LPIbaPr06UyfPr3Yc9u3by/0/ZdffsmXX35pgaiqpl27dvHmm28yatSoYpOX6JQsXlp2DIDHezRgYGtfss5dr7ILhgRQrVbLfBdRrEmTJjF58uQyt1epVHx0f1uOR6dy6UoWs1ad5IsH25svQCGEuIUqt9qotittpVFegZ7pvx8lNTufdoGuvDbI0J1vLFR37tw5FEXhmWeeITc3l2+++cZygYtq43aGEjetXUXjpF2oVbD8SAwrj8aYITIhhCgbSV6qmNKSl0/WG+a5ONtq+XZsB6y1hv99jRo1AiA1NZXk5GQAtFptjZi8LMxLr9eXqc3o0aP55Yt3GNvGBYA3V57k0hWp/yKEqBySvFQxxuTF379wif9/j8Uyf7dhnstno9sR6H59voKdnZ1piMg4dCREaWbNmoWPjw9z5869ZVtjhWsHBwfeGtWVrvXdycgt4NmloeTrbp38CCFERZPkpYoprufl0MWrvPSXYZ7L1Lsacm+rojVyli1bxoULF+jatSsDBw5k3LhxJCUlWSZoUe3odDoSExPLtOJo586dAHTv3h1bG2u+fKg9LnZWHIu6xhebzpk7VCGEKEKSlypEUZQiyUtkciZTfjlMnk7PgFY+vPr/y1Zv1rVrVxo1akRWVhYbNmzgjz/+wNbW1mKxi+qldevWAJw8efKWbY3Jy1133cW5c+c4uG09H48yLLf+cUc4IZdSzBeoEEIUQ5KXKiQtLY3MTMM8goCAAFIy83h80SFSsgwTdL8a0wG1uvTJlsYaL66urjg5OZk9ZlE9GWu9nDxZeuVcRVFMyYu9vT3NmjVj4sSJ9GrgzP0dA9Ar8NKyY2Tn6SwStxBCgCQvFpeXl8fLL7/MhAkTKCgovF+Mg4MDJ0+eZNOmTWisbZny62EikzMJcLVj/oTO2FlrSrxvbGws77zzDg8//DAg2wKI0jVr1gwrKyvS09O5dKnkyrkXLlwgPj4eGxsbpk2bRsOGDUlLS+Ovv/7i7WGt8HW2JTI5k4/Xn7Vg9EKI2k6SFwt77bXX+Oyzz/j111/Zt29foXNarZZWrVrRt28/Xv77OIcupuBkq2XRY13wcrIp9b4ZGRm8++67hIaGAlKgTpTOysqK5s0NQ5ClzXvZsWMHAEFBQdjZ2TFlyhQAfvzxR1zsrPj4gbYALNp7kb3hyeWOQ6/XM2LECB5//PFyXyuEqL0kebGw1157zfT1zckLGLrpZ685zX/HYtGqVfzwcCea+Nx6+KdBgwZoNNd7ZqTnRdxKWbYJGDx4MAsXLuT5558HYOLEiWi1Wg4cOMCxY8fo3dSLsV0NP2uv/H2cjNzCvYlr1641rVYqTlhYGKtWrWLhwoVkZWXd4SsSQtQWkrxYQHh4uOlrb29vPv74Y6Bo8rJ+/XoefHs+C/dcBODzB9vRo7FnmZ7DysqKBg0amL6XnhdxK927d6dPnz7F1hQy8vf3Z+LEiYwcORIAHx8fRowYAcC8efMAeGNIC+q42RGdks37a67vVH3o0CGGDBlCUFBQife/du2a6eubd58XQoiSSPJiZnv37qV58+YsWrTIdKxHjx6AIXm5cbLkd+uOcijPUN9l1tCWDG9f8odKcZo0aWL6WnpexK08/fTTbNu2jUcffbRc102dOhWAX3/9laysLBxttHz6QDsA/jh4mZ3nDEv0w8LCAOjYsWOJ97oxYYmPjy9XHEKI2kuSFzP777//KCgoMP2VCtCpUyesrKxISEjg4sWLAGw+ncBxW8Py1SDnVB7v2aC425XKuE3A888/z+jRo+88eFGr7dixg6+++oozZ84UOt63b18aNmyIWq02DTkFN/JgYvf6ALyx8gTZeTrOnjVM4m3RokWJz2FMXvz9/Wnbtq0ZXoUQoiaS5MXMEhMTAcPcASNbW1s6dOgAwIEDBwi5dJWnfz8CKjUZxzcxtqXDbT2XseclIiICa2vrO4xc1BZpaWnFzjdZsmQJL7zwAgsWLCh0XK1W8++//xIbG1toSOilAc3wd7El6mo2X20+Z0p6SkteAgICGD58OM8++ywODrf3cy+EqH0keTEzY5Vbb2/vQsfnzJnD2bNn6dBnMI8vOkxugR591DGurJ9DnTrlGy4yMiYvN86xEaI0DzzwAC4uLqxcubLIOWN9l969exc516pVK+zs7Aodc7TR8t5wQ+/h/N2RnIpJBeCFF15g9+7dxT7/0KFDWblyJa+++uqdvAwhRC0jyYuZGXtebk5eunbtipNPXSYuPERqdj4dAl2J/Xs2KPpSJ1CWpkePHoSFhXH06NE7jlvUDl5eXkDRFUfx8fGEhYWhUqlMc7TKon9LH4a08UOnV0hrPgxUhl8xpSXUx48f54svvmDFihW38QqEELWRJC9mZkxejB8SRtey8nh0wUHiUnNo7O3I+wPqoMvLQaPRFEl0ysrBwYGmTZtiZWV1x3GL2sG4XPr48eOFju/atQuAtm3b4ubmVuy1M2fOpGXLlqxevbrQ8bfva4mjtRpr38Y4dRoGlDwZNzMzk927d/Piiy/y66+/3tFrEULUHpK8mFlxw0bZeTqeWHyY84kZWBdkMqOjNRlXDRMX/fz8CtVrEcKcSqr1cuN+RiWJiYnhzJkzRXr6vJ1sGdfSMKTk2usRNM7exMXFFXuP1q1b8/TTTwOy2kgIUXaSvJhRQUEB3bt3p0OHDqbkpUCn55k/jhJyKQWNLpeLi1/hxIGdtGvXjlOnTvH3339XctSiNjFu0BgVFcWrr75KbGwsULbkxbg66OZeG4DXHuxN1wbuqK1t8bj3KWKLSV4URSm0VFrqvAghykqSFzPSarVs2LCBI0eO4OzsjKIovPPfKTafScBaq2a4ezz5yZfYt28fNjY2tGzZstSCXkJUNDc3NwYOHAjAJ598wkMPPURGRoapRktpyUu7dobaLseOHStyTq1W8eH9bdCoFOwadSYy17FIm/T0dLKzs03fx8fHl7pJpBBCGEnyYkE/7Ijgt/2XUang6zHteaB3e6BosTohLGnNmjX8+++/9OzZkxdeeAFHR0euXr3Knj17Sp1/Zex5uXDhgmk3dCNFUWjk5cjghoYl+0mBfYrsPG0cJlKpDDulZ2VlkZGRUWGvSwhRc0nyYkY3JiSrQmNMO+++NaQlg9r40bFjR6ysrEhMTOSdd95h9uzZxXbBC2FOarWaYcOGsWvXLlPpf3t7e7p3717qdT4+Pvj4+KAoSqH9i/R6Pb6+vnTp0oURTe3RpSWBgzvfb79Q6Hpj8tK4cWNTjRcZOhJClIUkL2b0yy+/4Orqysipr/LSMkPX+qSeDUzVc21tbU2l09977z1mzZpVbBe8EJZi7AUpK2Pvy40/t5cuXSIxMZHjx4/Tu0c3fpx2LwA/7Izg0pXrPTTG5MXX1xdfX99Cx4QQojSSvJhRYmIiWVYuHHfpRr5OYUgbP14fXLjaaHBwcKHvb7fGixCVoUuXLrRt2xYbGxvTsdOnTwOG7Sq0Wi2DWvvRs7EneQV63v3vtKndjcnL4sWLOXTokKnytBBClEaSFzO6lHgN7wfeQae2pkt9Nz5/sB1qdeG/bIODgwstjZbkRVQn77//PseOHWPChAmmY8ZtAVq2bAkYenPeua8VVhoVW88msvm0YWioXr16jBgxguDgYHr06EHnzp1liwAhRJlI8mImmbkFbNe3QOvijYs6l58e6YytVdH6Lffddx8xMTGm7/39/S0ZphAV7uY9jd58800mjLiXu///R/vd1afIydcxfPhwVqxYwQsvvFBZoQohqilJXsxAp1d49o+jZFi5octKZUK9DNwcit8o0dbWlpSUFACcnJxwcnKyZKhCVIiCggLy8/OBoj0vZ8+eZd++fTQriMDX2bBx4087Iwpdf+LECT7//HP++ecfywYuhKiWJHmpYIqi8N5/p9hyNhF0+ST+M5tmAe6lXmPseZEhI1EdjRs3DicnJ9atW4eiKKY5L8aeFz8/PwCuJMQyc3BzAL7fHk5E3BXTiry9e/fy0ksv8csvv1TCKxBCVDeSvFSwBXsusnjfJQCUfYvIiz17y72KjMujz549a/b4hKhoGo2GnJwcjh8/TnZ2NnfddRdNmzY17XJuTF7i4+O5r50/Heq6kp2vo/czn2FnZ8fx48dltZEQolwkealA60/G8b81hr86Xx/cnI5eKjp27Gj65V2SRx99lEcffZRt27ZZIkwhKpSx0u7x48ext7fn33//JSwsDFtbWwBTYhIXF4dKpeLNIYbhJHWj7iiudfDw8MDHxweQOi9CiLLRVnYANUXIpRSeWxqKosDD3eoyuVdDpty1okzXuru7s2jRIvMGKISZFFfr5UbG5N24OWOnem7c28ydjWFXcbv7Cby8vEzzZYxbBJS33owQonaRnpc7sGDBApo1a8bKzXuY/Mthcgv09G3uzTvDWskvX1FrGHtezp8/T3JycpGtLm4cNjIa09wWfX4utvXasu38VVPPS25uLmlpaRaKXAhRXUnycpuWL1/OE088wYWoeGZtiedqZh5tAlyYM7YDWo28raL28PHxwdvbG0VRaNCgAR4eHqxdu9Z03tfXF2traxwcHNDr9QDoM5JJP7QSgA/XnkFjZYOzszMg816EELcmn7K3Yffu3YwbNw6V1po2T31LmmJLgKsdP0/sjIONYSTuv//+w9XVlVGjRlVytEKYn3HoKCMjg5SUFLy8vEznfHx8yMnJITw8HLXa8CsnPj6e1AN/o8nP5OKVLH7Zd1HmvQghykzmvJTTmTNnuO+++8jNy6ftk19zTeuOs62WRY91wdvJ1tQuMTGR1NRUcnNzKzFaISzj3nvvRafTmSadN2/e3HSuuCHU+Ph4lLxsGmac4rxbV77Zcp5vf1qAu6OtaYm1EEKURHpeyiE2NpaBAweSkpJCi4ffJtW5AdYaNT8+0pkmPoWLyyUmJgLccpm0EDXByy+/zP/+9z8AAgMDb1lssWHDhowcOZJhrb1o5uNEWk4BIdkeskWAEKJMJHkph1deeYXLly/T4L7pZPl3QqWCOtGbmDry7iJd3UlJSQCFus+FqMlu3hbgRu+//z7dunXjr7/+AmDUqFEsX76cZ5+ZzisDmwGwaM9F4lKzLRewEKLakuSlHObOnUvvSW+hbzEQgNnDWxO+/R9OnDhBeHh4obbS8yJqm1OnTgFQt27dIuciIyM5cOAA586dK3Kub3NvutZ3J7dAzxNfLGfZsmVmj1UIUb1J8lIOey5ncckzCIDn+jXh4W71aNiwIQAREYX3ajEmL9LzImqLL7/8EgB7e/si524sVAeQlpZmWlKtUql4dZBhjsypbCd+WvqvJcIVQlRjkryU0YGIKzz//0XoxgXV5fn+htLnJSUvxmEj6XkRtcWyZcsYOnQo7777bpFzNxeqa9iwIba2tqYtMTrVc6ONu4JKreGSSzvLBS2EqJYkeSmj+p4ONPRyYGArX2YPb21aQdGoUSOAIsNGbdq0oVOnTrLZoqg1HnjgAVOJgJvdWKguLy+PK1eukJeXh6enp6nNYx3dUfQ68rxbEHIpxVJhCyGqIVkqXUY+zrb89WQw1ho1GvX1pZ8l9bzI7rhCXHfjsJFxSFWr1eLufn3H9c5N65D5xZ84tr2Xj9ef5c8p3aRStRCiWNLzUg7OtlbYWmkKHSup50UIcd2NPS/GCro+Pj6monVgGGK9tvt39Pm5HIy8yvawpEqJVQhR9UnycocaNmyIg4NDoc3lhBCF+fr6YmNjg6+vrynRN1bUNbK1tcVJU0D6kdUAfLYxrMg+SUIIAZK83DF3d3fS09M5duwYVlZWAOzZswdnZ2f69etXydEJUTXY2dmRnZ1NZGQkGRkZwPWhpBv5+PiQduAfbDVwKjaNDadknyMhRFGSvNwhlUpVZFw+MTGR9PR0srOl4JYQRsZ/J8Zho+KSl0WLFnFw11YeDa4HwJebzqPXS++LEKIwmbBrBlKgToiSNWnShPvvv5+goKAi57p16wZA4+x8/jgcQ1hCOqtPxHFfO39LhymEqMKk56UC/PLLL7Rr147XXnsNkAJ1QhTniy++oFu3bqSnp/PPP/8wZcqUEtu62FkxuZdhJd9Xm89RoNNbKkwhRDUgyUsFyMnJ4fjx45w4cQKQAnVCFCcmJoYDBw6YCtMV58SJE3z66af8+eefPNazAW72VkQkZbIqNNaCkQohqjpJXirAzbVepOdFiKKMy6XPnj1b4iqiQ4cO8corr7B48WIcbbRM7W0oRfD1lvPkS++LEOL/SfJSAYy1XiIjI9Hr9TLnRYhiGCforl69GhsbmyKFHeH68mnjLu0Tguvh6WjN5atZ/BMSbblghRBVmiQvFSAwMBCtVktubi6xsbE0b96cTp06Fbu7rhC1lbHnBSA/P7/Ynsmbkxd7ay3T+jQGYM7WC+QW6CwQqRCiqpPkpQJotVrq1TMs7YyIiOC7777j8OHD9OzZs5IjE6LquHFptL29PY6OjiW2SUhIQK83DBOND6qLj7MNMdey+SckxjLBCiGqNEleKohx3otsEyBE8W7seXF1dS123yLjUGtBQQEpKYbNGW2tNDz5/3Nf5m67IHNfhBCSvFSU1q1b06pVK1OVXSFEYW5ubqavbW1ti21jbW1t2qzRWMwOYGzXung6GnpfVhyR3hchajtJXirIF198wcmTJ+nYsSPOzs507ty5skMSokpRqVR8++23ALRr167EdjfPewFj74uhd/PbbRek7osQtZwkLxXMuDWAcf8WIcR1xoSkuK0BjBYuXEhISEiRCrzjguri4WBYefTvMan7IkRtJslLBZMaL0KUrEWLFowaNYquXbuW2CYoKIiOHTvi4OBQ6Li9tZZJ/19199utF9DJnkdC1Fqyt1EFyc3NJSgoiGPHjgFS40WI4owdO5axY8fe9vWPBNfjx53hRCRnsvp4LMPbB1RgdEKI6kJ6XiqIjY0NMTHXJxJK8iLE7QkLC+PTTz9l4cKFRc452miZ1LMBYKj7IjtOC1E7SfJSgYzLpUGGjYS4XSdPnuSVV15h3rx5xZ6f0L0+zrZaLiRmsPZknIWjE0JUBZK8VCDjNgEgPS9C3C5jwceLFy8We97Z1orH/7/3Ze628BL3SRJC1FwWSV7mzp1L/fr1sbW1JSgoiIMHD5baftmyZTRv3hxbW1vatGnD2rVrLRHmHbux56VBgwaVGIkQ1ZcxeYmLiyM3N7fYNhO718fBWsOZuDS2hyVZMjwhap2q+AeC2ZOXP//8kxkzZvD2229z5MgR2rVrx4ABA0yrcm62d+9exo4dyxNPPMHRo0cZMWIEI0aM4OTJk+YO9Y4Ze17uvfdehgwZUsnRCFE9eXp6Ym9vD8Dly5eLbeNqb834boYkZ+62CxaLTYja5vXXXycwMJCwsLDKDqUQlWLmlCooKIguXbqYilPp9XoCAwN55plneO2114q0HzNmDJmZmaxevdp0rFu3brRv354ffvjhls+XlpaGi4sLqampODs7V9wLKYOdO3fSu3dvGjVqxIUL8gtViNvVsmVLzpw5w6ZNm+jfv3+xbRLTcuj58TbydHr+mhpM1wbuFo5SiJrPuI2HJT7XyvP5bdael7y8PEJCQgr98lGr1fTv3599+/YVe82+ffuK/LIaMGBAie1zc3NJS0sr9KgsjRo1okWLFrRt27ZKdrMJUV3cat4LgLu9liGtPADpfRHCXObMmQMY9u07ffp0JUdznVmTl+TkZHQ6nanct5GPj0+hfUtuFB8fX672H374IS4uLqZHYGBgxQR/GwICAjh9+jTLly8vdtM5IUTZ1K9fH4BLly6V2Obw4cN8+9R9oOjZcS6Jj378jdhYqbwrREWaPn06999/PwCzZ8+u5Giuq/arjWbOnElqaqrpERUVVdkhCSHu0IwZMzh69Cgvv/xyiW1OnjxJQWoCmad3AvD5upM0atSI6OhoS4UpRLWm15dtj7BZs2YBhjmsVaX3xazJi6enJxqNptAGa2DY36SkvU18fX3L1d7GxgZnZ+dCDyFE9dakSRPat29f6r/nSZMmceXKFf43/i4A7Jv1oMDeo8r8chWiKsvJycHW1pZ69eqVON3i3LlzzJ8/n8zMTEaMGIGiKPzvf/+zcKTFM2vyYm1tTadOndiyZYvpmF6vZ8uWLQQHBxd7TXBwcKH2AJs2bSqxvRCi9omLi0NRFNzd3Zn60DD6t/BGpVLjEjSKK1euVHZ4QlR5MTEx5Ofnk5SUhJOTU7Fttm/fzuTJk/nwww+ZNWsWrq6utGjRokrM6TT7sNGMGTOYN28eixcv5syZM0ybNo3MzEwee+wxACZMmMDMmTNN7Z977jnWr1/P559/ztmzZ3nnnXc4fPgw06dPN3eoQogqIi8vj08//ZTp06dTUFBQ6Fxubi516tTB2dmZ5ORkAJ66uzEADq36Ep9WfG0YIcR1xikW2dnZfPnll8W2MW55U6dOHTp06EBMTAxvvfVWlZjTafaNGceMGUNSUhKzZs0iPj6e9u3bs379etOk3MuXL6NWX8+hunfvzu+//86bb77J66+/TpMmTVi5ciWtW7c2d6hCiCpCq9Xy5ptvkpeXx8svv2xafQRw4cIF9Ho9arUaDw/DaqOOdd0IauDGgcgUMgNK3rFaCGFw4/zQQ4cOFdvGOH8sIMCwAaqx/lJVYJFdpadPn15iz8n27duLHBs9ejSjR482c1RCiKpKrVZTt25dLly4wMWLFwslL2fOnAGgRYsWhf4CnNanMQciD/HHwcs807cJLvZWFo9biOrixontJRWDNLapU6eO6ZiiKGzcuJGkpCQefvhh8wZZimq/2kgIUTOVtFz67NmzADRv3rzQ8d5NvWju60Rmno7fDpS8xFoIUbjnpaTk5cZhI6MLFy6Qm5vL+PHjzRvgLUjyIoSokkoqVFdS8rJ3714yQ1YBsHBPJDn5OvMHKUQ1dWPyEhsbS35+fpE2Nw8bgWEl4H333Vfp814keRFCVEnG5KWsPS8pKSns/v1r1NnXSM7I458jUu9FiJK0bt3atIpXr9ebelmMMjIySE1NBQr3vFQVkrwIIaqk4oaNFEUpMXnx8PAAvQ79mc0AzNsZgU5f+Us6haiKPvzwQ/bu3UvjxoaVejcPHVlbW7NlyxZ+++23EpdSVyaLTNgVQojyKm7YKDc3l6effpqwsDAaNmxYqL1x5dHVw6tp0GMsF69kseFUPIPb+FksZiGqm7p165KSklKkUJ21tTV9+/Yt9pqw+HS8nWxwc7C2RIjFMvuu0pZWmbtKCyEqTmZmJufPn6devXq4ubndsv2VK1fw9PQE4JO1p5i7I5J2dVxY+XSPSh+fF6Iq0ekM88E0Gg35+flYWZV9Zd7GU/E8/2co7eq48ssTXbHSVNwATpXZVVoIIW6Xg4MD7du3L1PiAuDq6mpKUoY0c8JGq+ZYdCr7IqTirhA32rFjB7a2tvTr16/ExGX37t3Mnz+f48ePA4Yh27nbLjD1txCy8nSo1VTqpHhJXoQQ1UZ4eDgJCQnFlifXaDSmREfJTmN0Z8Mkwx93RFg0RiGquqioKAoKCtBoNCW2+eOPP5g8eTLLli0jJ1/H83+G8umGMBQFHg2ux6LHuuJkW3m1lCR5EUJUWX/99RfTp09nx44dADz55JP4+vry66+/Ftvew8MDW1tb0tPTmdyrIWoV7DiXRFh8uiXDFqJKMy6TrlOnDmfOnGHAgAEMGzasUBvjMmkXn0DG/LSfVaGxaNUq/jeiNe8Ob12hw0W3QybsCiGqrDVr1vDLL78QEBBA7969TSuNmjRpUmz7EydOYGNjY/p+YGtf1p6IZ96uCD4b3c4iMQtR1RmTl8DAQNRqNRs3bsTR0RFFUUxDrzExMWgc3Vma7E9yzjVc7a34bnxHujfyrMzQTaTnRQhRZd1Y6yU9Pd3012CzZs2KbX9j4gIwuZdhRdKq0BjiU3PMGKkQ1ceNyUvdunUBQ12Xa9eumdpEJ13De8z/SM5REehux6qne1SZxAUkeRFCVGE31no5d+4cAN7e3ri7u5fp+g513ehS3418ncKivRfNFKUQ1cuNyYudnR1eXl7A9VovSamZqPs+i7VnXbwdrfl9UjfqeThUWrzFkeRFCFFl3VjrpaTidDdaunQp9913H999953p2JS7GgGw5MAlMnILzBitENWDsQczMDAQwNT7cumS4d/Ioz8fwNqnEbrMFP6Y0o1A96qzm7SRJC9CiCrrxp4X427SpSUv4eHh/Pfff4SEhJiO9WvuTUNPB9JzCvjzUFSJ1wpRGxQUFDBgwAC6d+9eJHkJvxTFpMWHOJ2YjS47DavdP9DIu+pV1wVJXoQQVVhgYCAqlYrs7Gx27doFlJ68GKvsXrlyvbaLWq1i0v/PfVmwO5J8nd6MEQtRtWm1WpYuXcqePXtMZf+NycuqaFv2R1zF0UbDu3d78d2Hb1ZmqKWS1UZCiCrL2toaPz8/YmNjCQ4OplOnTvTs2bPE9sYKuzcmLwD3dwzgi01hxFzLZu2JOIa3DyjuciFqpXr16uHT/X4uqnxRq+DHRzrTo3HVmZxbHElehBBV2rZt2/D09MTNze2WZf6NPS/JycmFjttaaZgQXJ8vNp1j3q4I7mvnL1sGiFopJycHKyurQgXq+o9+jO+vNCevQM+L9zar8okLyLCREKKKa9q0Ke7u7mVKNoobNjJ6uFs9bK3UnIxJY1+4bBkgaqePP/4YW1tbXnnlFQCuZeUxbckR8gr09GvuzbTejfjrr7+YN28ekZGRlRxtySR5EUJUeZcvX2bv3r1cvXq11HamnaWvXi2yhYC7gzWjOxkmKM7bJVsGiNrJuDWAo6Mjer3CjL+OEZ2STaC7HV882B61WsVXX33FlClTCk18r2okeRFCVGnHjx+nXr169OjRg6eeeqrUtsbkxdramvT0olsCPNGzASoVbAtL4nyCbBkgap8bl0l/vyOcrWcTsdaqsT38G906teXSpUvExMQAhu0DqipJXoQQVVpsbKzp69JWGgHY2tqSmZlJVlYWzs7ORc7X93Tg3pY+AMzfVXW7xIUwF2OBOlwD+HKTofDj7OGtuHh0F2fPniUyMtL0b06SFyGEuE3GWi9Q8rYAN7K3L72glnHLgBVHY0hKz72j2ISobqKiokCl5vcLagr0CgNa+TCmS13TcunDhw9TUFCAWq3G19e3kqMtmSQvQogqzfhLFa5XBL0Tneq50aGuK3k6Pb/uu3jH9xOiukhNTSU9PR2nTsM4fyUXJ1st7w1vDVyvZr13714AfH190Wqr7oJkSV6EEFWavb0948aNo2fPngQFBd2y/WeffcawYcPYsGFDsedVKpWp9+XX/ZfIztNVaLxCVFXR0dFoXXxwu2sCAK8PboGPsy1w/Y8EY/JSlYeMQJIXIUQ1sGTJEnbt2oWVldUt2x46dIjVq1eb9kIqzoBWvgS625GSlc/fR6JRFAW9XirvippNq9XS8tHZqKxsCGrgzpjO13syjclLQkICAAEBVbuQoyQvQogapbRaL0YatYonejQA4IvVR2nWvAVr1qyxSHxCVJZTmQ6k2vljrVXz4f1tUKuv104yJi+enp5s3bqVmTNnVlaYZSLJixCiRilL8gIwunMgTjYaUgqsiNa7snjxYkuEJ0SluJKRy+zVpwF4rl8TGno5Fjpft25dPDw8aNGiBXfffTddunSpjDDLTJIXIUSNUtbkxcFGS8ax9QA4d72ff//995bXCFFdfbbxHClZ+TT3dWLKXQ2LnG/atCnJycns3LmzEqIrP0lehBA1SkmbM94sOTmZqC2/oujysQ1shcqzAUuXLrVEiEJY1IXEdP48dBmAXV89w8rl/5TY9uuvv+ann366ZTXryibJixCiRilpc8abHThwAF3GVbQxoQA4dx0pQ0eiRvpo3Vn0CqjjTpJ16bgpwb+Zoii8/vrrTJ06lZSUFAtHWT6SvAghahRj8pKVlVVquwMHDgDQwd7wF6Z90+4cPXeZ06dPmzdAISzg4MGDbN++nf0RV9h8JhGNWkXylvlAyfWSZsyYYfp34+/vb7FYb4ckL0KIGqVjx45kZWURFhZWarv9+/cD0L9zS+5q6oVKrcGp83DpfRHVXnZ2Nn379uXuu/sy9TvDvK6Rbb1Jj7kAlFzD5cSJE6av7ezszB/oHZDkRQhRo2i12lv+4tXr9Rw8eBCAoKAgpvx/0TrXjoPoO3Co2WMUwpyOHDlCZmYm9s17kqp1xcFaw7D6ho97T09PbG1ti71Oo9FYMsw7IsmLEKLWOXfuHKmpqdjZ2dGmTRt6NPagua8TerUVF/Cr7PCEuCMHDhwAjRaPux8DYGrvRmRciQNK32Lj5ZdfBmDkyJHmD/IOSfIihKhxnn/+eYYNG8b58+eLPW9jY8Pzzz/PxIkT0Wq1qFQq0/LRRXsvklsgWwaI6mv//v04dRiM2tkbbycbJvVqQFpaGgDe3t4lXte/f3/Onz9fLVbdSfIihKhxNm7cyOrVqw076BajQYMGfPnll3z33XemY0Pb+uPjbENSei7jXvuCa9euWShaIcpHr9cTGhpKXl5esecPhITi0v0hAGbc0xR7ay1ubm5otVqGDi19WLRx48ZYW1tXeMwVTZIXIUSNU9ZCdTey1qqZ2L0+APtSHFi5cpU5QhPijrVr144OHTqYVszdKD4+nmtebdHYOVPP3Y4HOhkm5w4cOJCMjAymT59u6XDNQpIXIUSNU1rykp2dzfbt28nIyChyblxQPTRKAdZe9dgelmD2OIW4HW3atAFg8+bNRc6prGyp0/9RAJ7t1xSt5vrHvI2NjWUCtABJXoQQNU5pycvhw4e5++67admyZZFzLnZWtLK9BkBoTvGFvISoTNu2bWPHjh1A8cnLf6evklmgpq67PcPbV+1aLXdCkhchRI1TWvJi7Grv3LlzsdcOqG+FotdxzcaHE9Gp5gtSiNuwdetWYmNjAcPPsnEiLkBOvo6fdkYA8FSfRoV6XWqamvvKhBC1Vmn7GxmTl6CgoGKvbdOoDplnDJvT/bgz3EwRCnF74uLiTF/rdLpCGykuPXiJxPRc3G1VDG9Xs5f8S/IihKhxjD0v6enpRc4ZK+t269at2GsDAwNJO7AcgLUn4rh8pfRtBoSwJGPyotVqgetDR3kFeuZuOQfA5Q3zsdbW7I/3mv3qhBC10vjx48nOzmb58uWFjsfGxhIdHY1araZTp07FXuvv709+UiTZkUfQK/Dz7ghLhCxEmRiTl0ceeQS4nrz8cySapCwdBelXaGmbWq2q5d4ObWUHIIQQFa2k8ufGIaPWrVvj6OhYbBsbGxvWrVtHAq68vf0Kfx6O4rn+TXF3qPq1L0TNZ0xexo8fj6+vL/379ydfp+e77YZ9i9IOLie4b/HzuWoSSV6EELXGrYaMjAYOHIiiKPx1bjenYtP4dd8lnuvfxBIhClEinU5HYmIiAC1btqRfv34A/BMSTdTVbFS5GWSErido5oTKDNMiZNhICFHjZGZm8uijjzJs2DB0uuul/idOnMicOXMYN27cLe+hUqmY2rsRAIv3XSQnX7YMEJUrMTERvV6PWq02lflXFMU0sTxl/z8oBbklTkavSaTnRQhR41hbW/PLL78AkJKSYlp91KJFC1q0aHHL60NCQti6dStNmjWnjpsD0SnZLAuJ5pFu9cwatxCl8fLy4sKFCyQlJaHRaMjLy+OrPzdyLkGFrQbSjq4lICCAgICAyg7V7KTnRQhR41hZWeHs7AyUb4sAoy1btvDKK6/wz7K/mNSzAQDzd0Wg0yvFto+IiOCee+7h119/vf2ghbgFrVZLo0aNTMOeOp2OL9edAMAr/TxKbmat6HUBSV6EEDXUjYXqwsPDCQ4O5s8//yzTtYGBgQBERUXxYJdAXO2tuHQli7Un4opt//vvv7N582ZTb48QlhB+NQ+bum1R9Dr6Bao5cOAAb7zxRmWHZRGSvAghaqQbk5dvvvmG/fv3s2jRojJdW6eOYTO76Oho7K21pg0bv9sejqIU7n1RFIUlS5YAhhUgQpjLf//9xxtvvMHWrVsBQ28gQNbZXRzbu42uXbvSsWPHygzRYiR5EULUSMbkJTw8nJ9//hmAGTNmlOnaG5MXRVGY2L0+9tYazsSlsT0sqVDb0NBQzp49C8DatWsZNWpURb0EIQpZt24dH3zwAdu2bSP2Wjb/HTf0BKYdXMG2bdsoKCio5AgtR5IXIUSNZExePv74YzIzM2ndujX9+/cv07XGCY+5ubkkJyfjam/NuK51AUz1NIyMvS49evRg2bJlrFixotjKvkLcqfj4eAD8/PxYuCcSnV6hW0N38hLCSU1N5Z133qncAC1IkhchRI1kTF6Mv/BfeOEFVCpVma61trbGx8cHMPS+AEzq1RBrjZpDF1M4dPEqYJgw+ccffwDw0ksvERgYiKIoHD58uEJfixBwvUCdi6cvfxyMAmDqXY1Mk9PXrFlTabFZmiQvQoga6YMPPmDhwoUAeHt7l6m2y41unLQL4Otiy6hOhh6Z77YZel927txJbGwsrq6uDBo0yLTSw1jJV4iKZExeTue6kpFbQBNvR3o39WLHjh0MGjSIpUuXVnKEliN1XoQQNZKDgwPfffcdAE899VSJWwaU5LvvvkOr1dK0aVPTsal3NeLPQ1FsC0vidGwaWq2Wu+++myZNmmBjY0NQUBB///23JC+iwimKYkhe1Bo2XswHYHKvhqjVKtq3b8/atWsrOULLkuRFCFFjzZw5k7lz5zJt2rRyX9ulS5cix+p7OjC4jR+rj8fx/Y5w5oztxdatW9Hr9QCFel4URSnzMJUQt5KSkkJeXh72Le4iKbMAT0cbhnfwr+ywKo0MGwkhaiSVSsXIkSPZvHmzqZR6RZjWx7BlwJrjsVxMzgRArTb8Ku3UqRMajYa4uDjTXBkhKoJxyMi9m2E12yPd6mGjrdk7R5dGkhchhCjGpUuX+PTTT/nmm28KHW/l70KfZl7oFfhy/clC5+zt7Wnbti1NmzY1TRQWoiI0bdqUVXtOoPFuhLVWzfhudSs7pEolyYsQQhQjKiqKV155ha+//rrIuce6+gGw8lg8e0PPFjq3b98+wsLCih12EuJ2WVlZsT4yF4CR7QPwdLSp5IgqlyQvQghRjJsL1d0o8tAWsi+GotJoWR2RV+icjU3t/lAR5hGdksX6k4bevMd61q/cYKoASV6EEKIY/v7+qFQq8vLySEoqXFV3zZo1pO4x1HdZFhJN7LXsItfrdDrTRF4h7tTMBRvQK9DaU0tzX+fKDqfSmTV5uXr1KuPHj8fZ2RlXV1eeeOIJMjIySr2mT58+qFSqQo8nn3zSnGEKIUQRxRWqA9Dr9Wzbto3c6FO09NSSr1P4fnt4oWtHjhyJq6srx48ft2jMombKyC1gb4Lh64YFlyo3mCrCrMnL+PHjOXXqFJs2bWL16tXs3LmTKVOm3PK6yZMnExcXZ3p88skn5gxTCCGKdXOhOoDjx49z5coVHB0def2+9gD8eSiKuNTrvS8ZGRlkZGRIvRdRIf4+HIVObU3+lSi61ZNeFzBj8nLmzBnWr1/P/PnzCQoKomfPnsyZM4elS5cSGxtb6rX29vb4+vqaHsbSx0IIYUk3znsxMu7oe9ddd9GzqQ9dG7iTp9Pz444IUxuptCtul06nK/y9XmHh3osApB3+lwB/v0qIquoxW/Kyb98+XF1d6dy5s+lY//79UavVt/wHvWTJEjw9PWndujUzZ84kKyurxLa5ubmkpaUVegghREUorudly5YtAPTr1w+A5/o1AeD3g5dJSMsBJHkRt2ffvn14e3szYcIE07GtZxO5dCULfU4Gmae24ucnyQuYMXmJj48vUhhKq9Xi7u5eav2DcePG8dtvv7Ft2zZmzpzJr7/+ysMPP1xi+w8//BAXFxfTw/jLRggh7tSzzz7L0aNHmTlzpunYggULWLp0KSNHjgSgeyMPOtdzI6/geu+LMXk5c+aM/EElyszX15erV6/y66+/muaHzt9l+JlKD12Hkp+Lr69vZYZYZZQ7eXnttdeKTKi9+XH27Nlb36gEU6ZMYcCAAbRp04bx48fzyy+/sGLFCsLDw4ttP3PmTFJTU02PG/9CEkKIO9GoUSPat2+Pi4uL6ZiPjw9jxoyhQYMGgKGS73P9Db0vSw5cIiEtB29vb+rXr4+iKBw6dKhSYhfVg16vNy3Fr1+/vmkPrl27dnEiOpUDkVfRqCD9yGrs7e1xcnKqzHCrjHInLy+++CJnzpwp9dGwYUN8fX1JTEwsdG1BQQFXr14tV+Zo/AvmwoULxZ63sbHB2dm50EMIISypZ2NPOtdzI7dAz5yt5wEZOhK3lpmZyciRI/nqq68AQyJs3P18y5YtzN/9/z15flbo0q/g5+cn+2X9v3JvzOjl5YWXl9ct2wUHB3Pt2jVCQkLo1KkTgGkDM+M/6rIIDQ0FkHE+IYTFZWdnM2fOHGJiYvjyyy959dVXcXNz47HHHiv0O0mlUvHygGaM+Wk/Sw9GMblXQ4YOHUpWVlaF7qskapaJEyfy77//snnzZsaOHYuvry/9+vVjwYIFbNpzmEzrvgC8Mrwj7w+LIDMzs5IjrjpUys2lIyvQoEGDSEhI4IcffiA/P5/HHnuMzp078/vvvwMQExNDv379+OWXX+jatSvh4eH8/vvvDB48GA8PD44fP84LL7xAnTp12LFjR5meMy0tDRcXF1JTU6UXRghxR/Lz87GxsUFRFCIjI2nevDm5ubmcPXuWZs2aFWk/YcFBdp5LYkR7f756qMNtPedHH33E0qVLWb9+vcxvqMHi4uKoU6cOer2e7du307t3bwASEhLw9fXFtc9juASNIrihB39M6VbJ0VpGeT6/zVrnZcmSJTRv3px+/foxePBgevbsyU8//WQ6n5+fT1hYmGk1kbW1NZs3b+bee++lefPmvPjii4waNYr//vvPnGEKIUSxrKysTAnEX3/9RW5uLgEBATRt2rTY9q8MMCQ0q47Fcja+/BN1169fz8yZMzl27Bhr1669/cBFlffHH3+g1+sJDg42JS5gmFPVql1HnNoNAGBSrwaVFWKVVu5ho/Jwd3c39bIUxzihzSgwMLDMPSxCCGEJgYGBxMXFsXjxYgD69u1b4ryD1gEuDGnjx5oTcXy2IYx5Ezqb/opUq0v/WzEpKYmJEyeavo+IiCi5saj2fvnlFwAeeeSRIufajpjK3hxH/B013N3Mm/nz5xMREcGoUaNM0zBqO9nbSAghSmEsVHf69Gngen2Xksy4tykatYrNZxLxbN4FNze3UstDGP3xxx8kJCSYvpfkpeY6fvw4x44dw8rKijFjxhQ6V6DTc9muMQBP39MCtVrFX3/9xYcffsjJkycrI9wqSZIXIYQohTF5Merbt2+p7Rt5OfJAR8M1jsGGlSMxMTG3fJ5nn32W33//nbfffhuQ5KUm+/XXXwEYOnQo7u7uhc5tPJ1AdEo2bvZWjPr/n6O4uDhAFq7cyKzDRkIIUd3dWPiySZMmZSqE+Wz/Jqw4GkOebzNs63cgOjqaLl263PK6sWPHkpaWxrhx46hXr94dxS2qrscffxy1Wl1sL968/y9KNz6oLsePhmBrayvJSzGk50UIIUpxY8/LrXpdjAJc7Xi4myH5cOvzGJejoottl5eXxyuvvEJSUpLpmLOzM02bNsXGxuYOohZVWYsWLfj444+59957Cx0/fPEqRy9fw1qjJmnv3wQFBfG///2PK1euAJK83EiSFyGEKMW9995LaGgo8fHxfPDBB2W+bnrfxlgp+Vj7NGR3bEGxbdavX8+nn35Kr1690Ov1FRWyqKa+226oJH9/xwAG9ukBwLJlywDDyjcPD49Ki62qkeRFCCFK4e7uTrt27fDx8SkyP6HU6xysCXI0/MUcqqtLek5+kTZhYWEAdOjQodBqpIULF/LYY4/J6ssaJiMjgwkTJrB27doiyeqp2FS2nk1ErYInezciODjYtFUAGPY9kuq610nyIoQQZjKgoT35V6LJ19gyd1vR/dmMk3IbN25c6PimTZtYtGgRBw8etEicwjKWL1/Or7/+yvPPP18kEfn+/3tdhrb1p76nA7a2tvTs2dN0XgoWFibJixBCmEnrls1pmmVY3rpgdySXr2QVOm/ccLZhw4aFjhu/lxVHNYtxldEjjzxSKHmJSMpgzQnDpNxpfRqZjhsn9LZr185UF0YYSPIihBBm0qFDB7b+9g29mniSp9Pz4bozhc4bk5Obk5dGjQwfYMbkRlR/0dHRbNmyBYCHH3640Lkfd0SgKNC/hTct/K6XxTcmLxcvXizSO1fbSfIihBBmpFKpeHNIS9QqWHcynv0RhnkwBQUFXLp0CZCel9pg8+bNKIpCt27daNDgesn/2GvZLD9qWI321N2FE5SOHTvi6upKamoqR44csWi8VZ0kL0IIYUaKouBjq+PBTgEAvPffaXR6hejoaAoKCrC2tiYgIKDQNcael0uXLlFQUPxKJVG9nD17FqBIef95uyLI1yl0a+hOx7puhc5pNBq+//57du7cSfv27S0VarUgyYsQQphRz549cXNzo4M2BicbLafj0lhy4BLe3t5s3LiRhQsXFtn3yN/fH2trawoKCoiKiqqkyEVFMq4su3E38uSMXP44eBmAp+8ufljooYceolevXlhbW5s/yGpEkhchhDAj4/Lq1MQYXvr/Xac/WR9GWr6ae+65h3HjxhW5Rq1Wm4YWLl++bLlghdmkpqYChZOXhXsiycnX07aOCz0be1ZWaNWSJC9CCGFGxgq90dHRPNytHu0DXcnILeDd/06Vet3mzZvJzs6md+/elghTmNnWrVtJT083/f+8kpHLoj0XAXiqT2Op4VJOkrwIIYQZGZOXmJgYNGoVH97fBo1axbqT8bw+9w+io4vfOqBOnTqFipSJ6s/R0dG07cPcbeFk5uloE+DCvS19Kjmy6keSFyGEMKMbe14AWvg5M6mXYUjol1M57D0YUmmxicoRnZLFb/sNK81eGdgMtVp6XcpLkhchhDAj40qiG3tYnu/XFH1aIlpnb3anuRV73fnz53niiSeYOnWqReIU5jNv3jzuvvtufv75ZwC+2nyePJ2e4IYeMtflNknyIoQQZnTjsJFRTmYaSeu/BWB1WDonolOLXJebm8uCBQv466+/LBOoMJuDBw+yfft2Ll++zPmEdJYfMSSyrwxsJnNdbpMkL0IIYUZ16tRh+PDhjB8/Hp1OB0BkZCQ5kUfQRexHr8DLfx8jt0BX6DrjaqNr166RkpJi8bhFxblxmfRnG8PQKzCglQ8d6hbf6yZuTZIXIYQwI0dHR1auXMncuXPRaDTA9bL//gn78XCw5mx8Op9tCCt0nYODg2kzPtkmoHozJi9qrwZsOJWAWgUv3dvsFleJ0kjyIoQQFmYs+98k0IePR7UFYN6uSPZcSC7UTrYJqP5SUlJITEwEYPkFPQD3d6xDEx+nygyr2pPkRQghzExRFFJSUrh27RpwPRlp1KgR/Vv6MC6oLgAv/nWMa1l5putkg8bqz9jrUqfLvRy8dA1rjZrn+zep5KiqP0lehBDCzJ588knc3d2ZO3cuAG+++Sbr1683Vdd9c0gLGno6EJ+WwxsrTqIoCiA9LzVBWFgYKq01Nt0nAPBo93rUcbOv5KiqP0lehBDCzHx8DEXIjMulAwMDGTBggKlUvL21lq8eao9WrWLNiTiWHzGsTDImL8YeG1H95Obm4t9vIgW2rvg62/Jc/6aVHVKNIMmLEEKYWXHLpW/Wto6raTjh7X9PcTE5k9GjR5OVlcWyZcssEqeoeP1HjsO+0wgAZg1riaONtnIDqiEkeRFCCDO7sVBdfHw87777Ln/++WeRdtP6NKZLfTcycguY+msIerUVdnZ2ZXqOzz//nJ9++qlC4xZlk5OTw/fff09SUlKh44qiMGvVKfJ0eu5q6sWg1r6VFGHNI8mLEEKY2Y1bBJw4cYJ33nmH9957r0g7jVrFt+M64uVkQ1hCOi/+dcw0/6U04eHhvPTSS/z7778VHru4tS+//JKnnnqKoKCgQsdXH49j94VkrLVq3ruvlRSkq0CSvAghhJkZk5ekpCTOnDkDXJ/PcjMfZ1t+eLgTVhoV60/FM+otQ2n5Xbt2lXj/Y8eOARAfH1/Bkdcc2dnZTJ06lcWLF1f4vdetWwcYig8mJxuWu6fn5PPOqhMAuMUdpL6nQ4U/b20myYsQQpiZu7u7aTfh3bt3AyUnLwCd6rkxe3hrAI7k+3MgKpOTJ0+W2P7ECcOHZJs2bSoq5Brnk08+4aeffmLFihUVfu8bh/aMw4GfbzzHlawC8q/GkH1kVYU/Z20nyYsQQpiZSqXi0Ucf5emnnyYqKgq4XsOlJA91rcvD3eqCSoXnsJc4FhFXYltj8tKsWTO+/fZbxo8fX3HB1wC5ubnMmzcPgCFDhlT4/SMjIwFDQhoUFMSOc0ks2nsRgKubvqd5k8YV/py1nSQvQghhAT/++CPffvstubm5QOk9L0azhrYiwDobtY0D23TNSUzPKbadMXnx9vbmxRdf5Pfff2fr1q0VF3w199tvvxETE4O/vz8TJkyo0HsrikJWVhYA27Zto26z1rz4VygAdXIiybkYSvPmzSv0OYUkL0IIYTGKopiq5ZYlebHWqpnewY6Ca/HkWrsw4eeDpGblF2qTlZXF+fPnARg8eDBTp04FDIXwyjLZt6bT6XR88sknAMyYMYOsrCz2799fYfdXqVRER0eTlpaGv38AM/48RnJGHs19nVAdMwxRGev5iIojyYsQQliAMXFJS0sDru8afSvd2rck4c+30GVc5Wx8Oo8tOkhWXoHp/OnTp1EUBS8vL3x8fJg5cyZ2dnbs27fPNJG0NluxYgXnzp3Dzc2Nnj174uHhwYABA0w7fFcUJycn5u2OZPeFZDTo+Oah9pw/cwqQ5MUcJHkRQggL+O6772jSpAlt27Zl06ZNZa7fUr9+fTxtFRL+fAsHKxVHLl9j6q8h5BYYPny1Wi33338/Q4YMQaVS4efnx/Tp0wFD74terzfba6rqFEXhww8/BGD69Ol07twZJycn0tLSCA0NrdDnOnI5hc82nAMgcd1cwg7tICEhAYCmTaWqbkWT5EUIISzA398fAHt7e/r371/m61QqFcHBwfjY6pjSLB97aw27zifz3B+hFOj0tG/fnn/++YeFCxearnnllVdwcnLi6NGjPPPMM2RnZ1f466kOrly5gkqlws7OjmeffRaNRsNdd90FwPbt24u0z83NZeDAgTRt2pR7772XpUuX3vI55syZQ79BQ3li3i50ioJ3ThQZxzcyZ84cunfvTuvWrXF2dq7ol1brSfIihBAWcGOhuvJasmQJUVFRPPfwcOZN6Iy1Rs36U/E8/fsRcvKLDn94enoyc+ZMALZu3Yqtre2dBV9NeXp6cujQIY4dO4anpycAffr0AYpPXpYvX86GDRs4f/48mzZtIjY29pbPse/AIU66dCclX0Ogux1vDzb0shw8eJAtW7aYJlOLiiXJixBCWMCNycvRo0fLde2NQ0w9Gnvy3fiOWGvVbDiVwIPf7SQ1K6/INa+99hqrVq3ik08+MVV2zc7OZvjw4RU+36OypaSkMHToUIYOHcpLL73E/Pnz2bVrFxkZGahUKpo0aWJqe/fddwOwc+fOIu/Djz/+CMDkyZNZuHAhAwcOBCA/P7/YpFOvVzhq0wa7+u2wUSt8P74Tg/r3ITAwkNTUVP777z9zvWSh1DCpqakKoKSmplZ2KEIIYVJQUKAACqB89NFHt3UPvV6v5OfnK4qiKPvCk5WWb61T6r26Wqkz6TvlUuKtf+c999xzCqCcOXPmtp6/qpo/f77pvTU+OnXqpCQnJxdpW1BQoLi4uCiAcvjwYdPxM2fOKICiVquVqKgo0/H9+/cr1tbWSqNGjQrdR6/XK2+vOqnUe3W1UvelFcr81btN52bOnKkAyrBhw8zwamuu8nx+S8+LEEJYgEajMX3dtm3bcl//2muv4eXlxR9//AFAt4YevNxRQ0HGVTQedRn782HCkzJKvYex9ktERES5n78qO3v2LGAYEnr++ecZOHAgV65coV+/fmRmZhZqW9K8F+OmlkOGDDH1koFhSXteXh7h4eFkZFx/f3/YEWEqRJe85ktGBrcwnXvkkUcA+O+//9i3b1/FvVBhIsmLEEJYyLZt2/jyyy9NwxHlUVBQwJUrV9i7d6/p2LWLp0j47WVs8lKJuZbNyLl72Hw6ocR7NG5sqPRqrDVTU4SFhQHwwAMP8OWXX7Ju3ToiIyMJDQ3FwaHonkLTpk1j3rx5jB492nQsMTERlUplqpNj5OXlha+vYTfoU6cMS5//Donm4/WGhOnqlnloY0Jxc3MzXdOiRQtGjhxJ3bp1adGiBaLiSfIihBAWYuwZuJ3dhbt37w5QKHk5ceIEBakJjHAMp0NdV9JyCpj0y2E+XHuGfF3RJdLGwng1reflm2++YfXq1QwdOrRM7QcNGsSkSZOoW7eu6dhvv/1GREREsYmlsafs+PHjLNoTyct/GzbC7F8H0g+vomHDhkX+n/7zzz9cunQJV1fX23xVojSSvAghRDUQHBwMGBIWY6E740qWLm1b8OeUYB7vYSh89+POCB76aT9xqYWXSBv3U6ppPS/169dnyJAh1KtX747vc+PwnpFhw0sVS8/m8s5/p1EUGB9Ul74e6Xh4eBRbcPB2ElRRdpK8CCFENeDn50eDBg1QFIUDBw6g0+lMwxht2rTBWqtm1rCWfD++I042WkIupTDkm92sOxFn2iagpva83I6IiAjmzJnDjz/+eMvl6y1at8XzvpcJtzIkKS8PaMb/RrRm3NiHSE5ONu0kLSxHkhchhKgmbhw6ioiIICsrC1tbW9NcFoBBbfxY/WxPWvk7czUzj2lLjjBp8WGiU7JMPS8RERE1Zt+jI0eO8N5777Fp06ZyXbdy5UqeffZZnnzySerVq2eqxHuzlMw8/rnih0OLu1B0BXzxYDuevrtxoZ4VKyurO3oNovwkeRFCiGrixuTF1taWV155hcmTJxcZ6qjn4cA/07rzbN/GWGlUbDmbyD1f7GRTlJ5pT0/n/fffJz8/v7inqHa2bt3K22+/zc8//1yu64zF6gD0ekOl4pttO5vIvV/t5HRSHhp9HkMcLzK0tfcdRiwqgrayAxBCCFE2PXv25K677uKuu+4iMDCQjz/+uMS2tlYaZtzbjPva+/P68pMcvHiVjzecp1mj0Qy/t2mN6S0wrjQq7+aH7dq1w8XFhdTUVOrVq8e9995rOpeRW8D7a07zx8EoABp5OTB3fC+a+14v868oCh07dsTHx4dffvkFb29JaixJkhchhKgm2rZty44dO8p1TWNvJ5ZO6cbfIdF8sO4MYQnpTP01hNYBzsy4pyl3N/Ou1pNLbzd50Wg09OvXj+XLlzNlyhRT79X+iCu8/Pcxoq4aJjs/3qMBrwxshq1V4d6t5ORk0+aOsneR5amUmjLw+f/S0tJM2bT8QAkhaqq9e/fSpEkTvLy8ynzNtaw85m4+y68Ho8kpMPzqbxfoypN3NaR/Sx+sNNVvJoGPjw+JiYkcPnyYTp06levamJgY1q5dy8SJEwm/ks3nG8+x6f/r5AS42vHZ6HYEN/IADENLFy9eJCUlhU6dOnHw4EGCgoIICAi4rf2qRFHl+fyufj+pQghRy8XFxdGjRw+8vb1JSCi5KN3NXO2t0Zxey/kvxlIn/Qx2VhqORV1j2pIj9PhoK19sOldkeXVVdu3aNRITEwFo2rRpua8PCAjgnpHjeOmfkwz6ehebTiegVsHYrnVZ/3wvU+ICsHbtWho1asRjjz0GXF+xVdwyaWF+MmwkhBDVyI4dO0yTTT09Pcs916JRo0bos9PQH13Ozg3PsGhvJH8eiiIxPZdvtpxn7rYL3N3Mi6Ft/enbwhtn26o7N8Y4ZOTv74+Tk1OZr9PrFfZFXGHpoSjWnYijQG/ohRrSxo8X7mlKY2/HItcYar3AmTNnyMvLMyUvxuXnwrIkeRFCiGrkxn2R/Pz8yj1f5cZaL15ONrw8oDnP9WvKhlPx/Lb/Egcir7L5TCKbzyRirVHTs4kng1r7cndzbzwdbSr0tdyp8s53iU/N4e+QKP48HGWa0wJwdzMvXry3Ga0DXEq8tm7dujg7O5OWlkZYWBiRkZGAJC+VRZIXIYSoRm7cQycvL6/c1xtrvSQkJJCRkYGjoyPWWjXD2vkzrJ0/5xPS+fdYLGtPxBGelMnWs4lsPWsYmmni7Uj3Rh4EN/IgqIEHbg7WFfOibtNDDz1Ely5dSnwfCnR6QqOusT0sie3nEjkZk2Y652SjZUSHAMZ0CSw1aTFSqVS0bduW3bt3c/z4cRk2qmSSvAghRDXzzjvv8M477/DNN9+U+1pXV1fc3NxISUkhMjLSNBxi1MTHiRfvbcaL9zbjfEI6a0/Es/F0PKdi0zifmMH5xAwW77sEQKC7Ha39XWgd4EIrf2ea+jjh62yLWm2Z1UvW1taFNj5MSs/lRMw1jkenciI6lcOXUkjNLlzPpkt9Nx7qUpfBbfywsy66FUBp2rRpw+7duzlx4gROTk64u7tLz0slkdVGQghRzeh0OpKTk/Hx8bmt6zt37kxISAgrV65k+PDhhc5lZGSQnp6On59foeNXM/M4EHGFfRFX2Bd+hfOJGcXe20arpp6HPfU9HKjv6YC3kw3ezrZ4Odrg7WyDu701jrbacq9sKtDpuZadT0pmHlcy80hIy+HylSwuXc3i8tUsLl3JJCEtt8h1LnZW9Griyd3NvLmrqRdeTrc/9PXDDz8wbdo0Bg0axNq1awFDvZfqvNS8KinP57f0vAghRDWj0WhuO3EBw9BRSEhIsRs0Dhw4kJCQEPbs2UPHjh1Nx90drBnUxo9BbQxJTWpWPqfiUjkVk8bJ2FROxqRy6UoWuQV6ziVkcC6h+OTGyEarxslWi4ONFo1ahVatQq1SoVGrUBTILdCRW6Ant0BPTr6O9JyCW74ulQoaeTnSNsCFNnVcaBfoStsAF7QVtATc2Et1/PjxG55TEpfKIMmLEELUMuPGjaNr16707du30PHLly+zZ88ewLDDcmlc7K3o3siT7o08TccKdHpirmVz8UoWF5MzuXw1i8T0XBLTckhKzyUxPZeMXEMSklugJzcjj+SM8s3bcbW3MvTeaBX2blqFkpbE/K8/or6nI018nHC0Md/HWps2bXj99ddp06aN9LhUMklehBCilrl5qMho/fr1APTo0QN3d/dy31erUVPPw4F6Hg70blp88bwCnZ7MXB3puflk5BaQmVtAgU5Bp1fQKQoFegUVhu0NbLRqbLQabKzUuNpZ4WJnZepFWb9+Pf+98DUtW7ZkZMfAcsd6O5ydnXn//fdZvHgxLVq04MEHH+S9996zyHOLwiR5EUIIAcC6desAw9ARQH5+PhkZGYVWON0prUaNi70aF/s7qx9zu9sCVISwsDDCwsK4evWqxZ9bGEiFXSGEqGX0ej1Hjx7ln3/+QafTAYZl11u2bAEMycvRo0fp1KkTjz/+eGWGWqLKSl6uXbvGDz/8AMgy6cokyYsQQtQyiqIQFBTEAw88QExMDAD79u0jPT0dLy8vOnbsiJWVFWfOnGHlypX8+++/lRxxUZWVvGzcuJGUlBRACtRVJklehBCiltFoNKYJucYVR8YhowEDBqBWq2ndujUvvvgiANOnTycjo/TVQ5Z27tw5wPLJy411cerVq2fR5xbXmS15ef/99+nevTv29va4urqW6RpFUZg1axZ+fn7Y2dnRv39/zp8/b64QhRCi1rpxmwCAIUOG8PTTTzNmzBhTm1mzZlG/fn2ioqJ48803KyXO4mRnZxMbGwtYPnlp0qRJsV8LyzJb8pKXl8fo0aOZNm1ama/55JNP+Oabb/jhhx84cOAADg4ODBgwgJycHHOFKYQQtZJxmwBjz0uvXr349ttvGTp0qKmNvb093333HQBff/11lRk+srOzIzMzk5MnT97Wqqg7odVquXz5MuHh4eXaDFJULLMlL++++y4vvPBCkdLTJVEUha+++oo333yT4cOH07ZtW3755RdiY2NZuXKlucIUQohayZi8GHteSjJo0CCef/55AB599FEuXrxo5sjKxtbWllatWlXKcwcGBsp8l0pWZea8REZGEh8fT//+/U3HXFxcCAoKYt++fSVel5ubS1paWqGHEEKI0t04bPT777+zc+dO8vPzi2378ccf07VrV9q3b4+trW2J96xhu82IKqzKJC/x8fEARUpe+/j4mM4V58MPP8TFxcX0CAy0TLEiIYSozow9L+fOnWP69On07t2bQ4cOFdvW2tqaNWvWsHnzZnx9fQudi46O5rnnnsPBwYGxY8eaPW6A2bNnM2nSJA4ePGiR5xNVT7mSl9deew2VSlXq4+zZs+aKtVgzZ84kNTXV9IiKirLo8wshRHXUqFEjPv30UyZPnkxKSgpubm507dq1xPaenp5oNNd3Yd6zZw9Tp06lYcOGfPPNN2RlZfHnn3+W+sdmRTh37hwLFizg559/Ji4uzqzPJaquclXYffHFF5k4cWKpbW53HNCYzSckJBTazTQhIYH27duXeJ2NjQ02Nre/S6gQQtRG9vb2vPTSS8yaNQuAe+65B6321h8Jubm5PPfcc/z444+mY7179yYxMZHp06djZ2dnlngTEhJ47733+PHHH9HpdLi4uNCtWzezPJeo+sqVvHh5eeHlVfx+FXeqQYMG+Pr6smXLFlOykpaWxoEDB8q1YkkIIUTZGfczGjRoUJnaq9VqU4G4AQMG8MYbb9CrVy+zxZednc1nn33GJ598Yqo1M3ToUD766KM72llbVG9m29vo8uXLXL16lcuXL6PT6QgNDQWgcePGODo6AtC8eXM+/PBDRo4ciUql4vnnn+d///sfTZo0oUGDBrz11lv4+/szYsQIc4UphBC11uHDh03zXAYMGFCma6ysrNi4cSMJCQnUqVPnjp7fuCVBt27dStw/6b///jP1DnXp0oVPPvmEPn363NHzihpAMZNHH31UAYo8tm3bZmoDKAsXLjR9r9frlbfeekvx8fFRbGxslH79+ilhYWHlet7U1FQFUFJTUyvolQghRM00ePBg0+/mihAdHa3MnTtX2bFjR5nav/XWWwqgODg4KC+99JISExNTpI1er1emTJmiLF26VNHr9RUSp6iayvP5rVKUmrW2LS0tDRcXF1JTU3F2dq7scIQQoso6deoUw4cPZ/bs2RWyUujll1/ms88+Y/z48fz222+3bD979mxTrwoYVjVNmDCB1NRUfvzxxwrdzVpUfeX5/JbkRQghRIXYvXs3vXr1wsXFhaSkJKysrG55jV6vZ82aNXzyySfs3r3bdPyJJ55g/vz55gxXVDGSvEjyIoQQFqfT6fDz8yMpKYlNmzYVKjpaFrt37+aLL75Ap9Px9ddfmzaPFLVDeT6/q0yROiGEENWbRqPhvvvuA2DVqlWltg0LC0On0xU61rNnT5YvX86qVaskcRGlkuRFCCFEhRk+fDgAK1euLHG7gJycHDp06ICvry/R0dGWDE/UEJK8CCGEqDD9+/fH3t6e6Ohojhw5UmybHTt2kJ2djY2NDQEBARaOUNQEkrwIIYSoMHZ2dgwYMACtVsvx48eLbbN27VrAUBhPpVJZMjxRQ5itSJ0QQoja6fPPP2fBggW4uroWe96YvAwePNiCUYmaRJIXIYQQFapBgwYlnjt//jwXLlzAysqKfv36WTAqUZPIsJEQQgiz0Ov1rF69utDE3XXr1gHQq1cvKWchbpskL0IIISqcoiiMGjWKYcOGFSo2d+N8FyFulyQvQgghKpxKpSIoKAiA6dOnc/jwYQDefvttXn/9ddOSaiFuh1TYFUIIYRaKojBy5EhWrVpFvXr1CAkJwcPDo7LDElWUVNgVQghR6VQqFYsWLaJRo0ZcunSJhx9+GL1eX9lhiRpAkhchhBBm4+rqyj///IOdnR3r169n8ODB5OXlVXZYopqT5EUIIYRZtWvXju+++w6AnTt3kp+fX8kRiepO6rwIIYQwu4kTJ+Lk5ISHhwcODg6VHY6o5iR5EUIIYRGjRo2q7BBEDSHDRkIIIYSoViR5EUIIIUS1IsmLEEIIIaoVSV6EEEIIUa1I8iKEEEKIakWSFyGEEEJUK5K8CCGEEKJakeRFCCGEENWKJC9CCCGEqFYkeRFCCCFEtSLJixBCCCGqFUlehBBCCFGtSPIihBBCiGqlxu0qrSgKAGlpaZUciRBCCCHKyvi5bfwcL02NS17S09MBCAwMrORIhBBCCFFe6enpuLi4lNpGpZQlxalG9Ho9sbGxODk5oVKpbvs+aWlpBAYGEhUVhbOzcwVGKG4m77VlyfttOfJeW46815ZjrvdaURTS09Px9/dHrS59VkuN63lRq9XUqVOnwu7n7Ows/xAsRN5ry5L323LkvbYcea8txxzv9a16XIxkwq4QQgghqhVJXoQQQghRrUjyUgIbGxvefvttbGxsKjuUGk/ea8uS99ty5L22HHmvLacqvNc1bsKuEEIIIWo26XkRQgghRLUiyYsQQgghqhVJXoQQQghRrUjyIoQQQohqpVYnL3PnzqV+/frY2toSFBTEwYMHS22/bNkymjdvjq2tLW3atGHt2rUWirT6K897PW/ePHr16oWbmxtubm7079//lv9vxHXl/bk2Wrp0KSqVihEjRpg3wBqkvO/1tWvXePrpp/Hz88PGxoamTZvK75FyKO/7/dVXX9GsWTPs7OwIDAzkhRdeICcnx0LRVk87d+5k2LBh+Pv7o1KpWLly5S2v2b59Ox07dsTGxobGjRuzaNEis8eJUkstXbpUsba2VhYsWKCcOnVKmTx5suLq6qokJCQU237Pnj2KRqNRPvnkE+X06dPKm2++qVhZWSknTpywcOTVT3nf63Hjxilz585Vjh49qpw5c0aZOHGi4uLiokRHR1s48uqnvO+1UWRkpBIQEKD06tVLGT58uGWCrebK+17n5uYqnTt3VgYPHqzs3r1biYyMVLZv366EhoZaOPLqqbzv95IlSxQbGxtlyZIlSmRkpLJhwwbFz89PeeGFFywcefWydu1a5Y033lCWL1+uAMqKFStKbR8REaHY29srM2bMUE6fPq3MmTNH0Wg0yvr1680aZ61NXrp27ao8/fTTpu91Op3i7++vfPjhh8W2f/DBB5UhQ4YUOhYUFKRMnTrVrHHWBOV9r29WUFCgODk5KYsXLzZXiDXG7bzXBQUFSvfu3ZX58+crjz76qCQvZVTe9/r7779XGjZsqOTl5VkqxBqlvO/3008/rfTt27fQsRkzZig9evQwa5w1SVmSl1deeUVp1apVoWNjxoxRBgwYYMbIFKVWDhvl5eUREhJC//79TcfUajX9+/dn3759xV6zb9++Qu0BBgwYUGJ7YXA77/XNsrKyyM/Px93d3Vxh1gi3+16/9957eHt788QTT1gizBrhdt7rf//9l+DgYJ5++ml8fHxo3bo1H3zwATqdzlJhV1u38353796dkJAQ09BSREQEa9euZfDgwRaJubaorM/GGrcxY1kkJyej0+nw8fEpdNzHx4ezZ88We018fHyx7ePj480WZ01wO+/1zV599VX8/f2L/AMRhd3Oe717925+/vlnQkNDLRBhzXE773VERARbt25l/PjxrF27lgsXLvDUU0+Rn5/P22+/bYmwq63beb/HjRtHcnIyPXv2RFEUCgoKePLJJ3n99dctEXKtUdJnY1paGtnZ2djZ2ZnleWtlz4uoPj766COWLl3KihUrsLW1rexwapT09HQeeeQR5s2bh6enZ2WHU+Pp9Xq8vb356aef6NSpE2PGjOGNN97ghx9+qOzQaqTt27fzwQcf8N1333HkyBGWL1/OmjVrmD17dmWHJipArex58fT0RKPRkJCQUOh4QkICvr6+xV7j6+tbrvbC4Hbea6PPPvuMjz76iM2bN9O2bVtzhlkjlPe9Dg8P5+LFiwwbNsx0TK/XA6DVagkLC6NRo0bmDbqaup2faz8/P6ysrNBoNKZjLVq0ID4+nry8PKytrc0ac3V2O+/3W2+9xSOPPMKkSZMAaNOmDZmZmUyZMoU33ngDtVr+dq8IJX02Ojs7m63XBWppz4u1tTWdOnViy5YtpmN6vZ4tW7YQHBxc7DXBwcGF2gNs2rSpxPbC4Hbea4BPPvmE2bNns379ejp37myJUKu98r7XzZs358SJE4SGhpoe9913H3fffTehoaEEBgZaMvxq5XZ+rnv06MGFCxdMCSLAuXPn8PPzk8TlFm7n/c7KyiqSoBgTR0W29KswlfbZaNbpwFXY0qVLFRsbG2XRokXK6dOnlSlTpiiurq5KfHy8oiiK8sgjjyivvfaaqf2ePXsUrVarfPbZZ8qZM2eUt99+W5ZKl1F53+uPPvro/9q3Q1dFoiiO47PlaplqU2EGLBaTxvkvbDJdrIJtDFot8rJREKMWi01sE8UiaDEaJir8Nj3Z3bdhJzzlsN8P3DR34NzDMPwY5sg5p+Vyqev1+lxZlr3rCGbk7fWfmDb6d3l7fblc5Pu+er2ejsejVquVSqWSRqPRu45gSt5+J0ki3/c1n891Op202WwUhqHa7fa7jmBClmVK01RpmsrzPE0mE6VpqvP5LEkaDAbqdDrP/Z+j0v1+X4fDQR8fH4xKf7fpdKpKpSLnnJrNpvb7/fNaFEWK4/i3/YvFQrVaTc451et1rdfrF1dsV55eV6tVeZ73ZSVJ8vrCDcr7XP+K8JJP3l7vdju1Wi0VCgUFQaDxeKzH4/Hiqu3K0+/7/a7hcKgwDFUsFlUul9XtdnW73V5fuCHb7fav79/P3sZxrCiKvtzTaDTknFMQBJrNZt9e5w+J72cAAMCO//KfFwAAYBfhBQAAmEJ4AQAAphBeAACAKYQXAABgCuEFAACYQngBAACmEF4AAIAphBcAAGAK4QUAAJhCeAEAAKYQXgAAgCk/AW0kcgS6gm/UAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAACHM0lEQVR4nOzdd3hTZf/H8XeS7j3ohLJHGWVDKUMQkDIFRB8EFHCA+sijAi5+Im7c4kRcCChbGQpY9t6rbMpoKd0Duneb8/sjJlDaQgtN0vF9XVcu6Tn3Sb6N0Hx6n3uoFEVREEIIIYSoJtTmLkAIIYQQoiIkvAghhBCiWpHwIoQQQohqRcKLEEIIIaoVCS9CCCGEqFYkvAghhBCiWpHwIoQQQohqRcKLEEIIIaoVC3MXUNm0Wi2xsbE4OjqiUqnMXY4QQgghykFRFDIyMvD19UWtvn3fSo0LL7Gxsfj5+Zm7DCGEEELchaioKOrVq3fbNjUuvDg6OgK6b97JycnM1QghhBCiPNLT0/Hz8zN8jt9OjQsv+ltFTk5OEl6EEEKIaqY8Qz5kwK4QQgghqhUJL0IIIYSoViS8CCGEEKJaqXFjXoQQQtQMRUVFFBQUmLsMUYksLS3RaDT3/DwSXoQQQlQ5mZmZREdHoyiKuUsRlUilUlGvXj0cHBzu6XkkvAghhKhSioqKiI6Oxs7ODg8PD1lwtIZQFIWkpCSio6Np1qzZPfXASHgRQghRpRQUFKAoCh4eHtja2pq7HFGJPDw8uHLlCgUFBfcUXmTArhBCiCpJelxqnsr6fyrhRQghhBDVioQXIYQQQlQrEl6EEEIII+rTpw8vvfSSucuoUSS8CCGEEFXEjh07UKlUpKammruUKk3Ci6hVFEXhxx9/5OTJk+YuRQghxF2S8CJqleXLl/PMM8/Qrl07c5cihKigrKysMh+5ubnlbpuTk1Outndb4/jx43FwcMDHx4fPP/+82PnffvuNzp074+joiLe3N2PHjiUxMRGAK1eucP/99wPg6uqKSqVi4sSJAISEhNCzZ09cXFxwd3dn6NChXL58+a5qrAkkvIhapUGDBoY/3+0PJyGEeTg4OJT5GDVqVLG2np6eZbYdNGhQsbYNGzYstd3deOWVV9i5cydr165l06ZN7Nixg2PHjhnOFxQU8N5773HixAnWrFnDlStXDAHFz8+PP//8E4CwsDDi4uL46quvAN3Pq2nTpnHkyBG2bt2KWq1m5MiRaLXau6qzupNF6kStEhQURN26dYmJieHYsWP06tXL3CUJIWqIzMxMfvnlF37//Xf69esHwMKFC6lXr56hzZNPPmn4c+PGjfn666/p0qULmZmZODg44ObmBujCl4uLi6HtreFs/vz5eHh4cPbsWdq0aWPE76pqkvAiap3OnTsTExPDkSNHJLwIUY1kZmaWee7W1Vr1t2JKo1YXv+lw5cqVe6pL7/Lly+Tn5xMYGGg45ubmRosWLQxfHz16lLfffpsTJ06QkpJi6Dm5evUqrVq1KvO5L168yKxZszh48CDJycnFrpPwIkQNt2LFCsOfDx8+bMZKhBAVZW9vb/a29yIrK4vg4GCCg4NZvHgxHh4eXL16leDgYPLz82977bBhw2jQoAE//fQTvr6+aLVa2rRpc8fraioJL6LGyMrKwtbWtsRvVXppaWmMHj3a8LWEFyFEZWrSpAmWlpYcPHiQ+vXrA5CSksKFCxfo3bs358+f59q1a3z00Uf4+fkBcOTIkWLPYWVlBeg2p9S7du0aYWFh/PTTT4be4j179pjiW6qyZMCuqJYURSnxG4e/vz/W1tZ8++23pV5z4cIFACwtLQGIiooiIyPDuIUKIWoNBwcHnnrqKV555RW2bdvG6dOnmThxouEXqvr162NlZcU333xDeHg4f/31F++9916x52jQoAEqlYp169aRlJREZmYmrq6uuLu78+OPP3Lp0iW2bdvGtGnTzPEtVhkSXkS1dOnSJVxcXBg0aBCKogC6++GFhYUsWLCg1Gv04SUoKIjQ0FDS09NxdHQ0VclCiFrg008/pVevXgwbNoz+/fvTs2dPOnXqBOh2VF6wYAErV66kVatWfPTRR3z22WfFrq9bty7vvPMOr7/+Ol5eXkyZMgW1Ws2yZcs4evQobdq0YerUqXz66afm+PaqDJWi/8lfQ6Snp+Ps7ExaWhpOTk7mLkcYyS+//MLTTz9Nz5492b17NwDnz5+nZcuWWFlZkZmZaehh0Zs1axbvvfcekyZN4scffzRH2UKIcsjNzSUiIoJGjRphY2Nj7nJEJbrd/9uKfH5Lz4uolnbt2gXAfffdZzjWokULnJ2dyc/P5+zZsyWuCQsLA6B58+amKVIIIYRRSHgR1VJp4UWlUtG+fXsAjh8/XuIa/W2jFi1akJCQwKRJkxg4cKDxixVCCFGpJLyIaicqKoorV66g0Wjo3r17sXMdOnQASoYXrVZbLLzY2dnxyy+/sHHjRuLj401TuBBCiEoh4UVUO/oxLh07diwx4FYfXm69baQoCitXruSLL76gUaNGODo60rJlS0CmTAshRHUj4UVUO6XdMtIbNmwYYWFhbNy4sdhxjUbD4MGDmTp1qmEgb5cuXQAJL0IIUd1IeBHVTmBgIIMHD+aBBx4occ7V1ZXmzZuXuVDdzfTh5dZFooQQQlRtssKuqHaeeOIJnnjiiQpds3HjRlJSUujRo4dhZcube14URUGlUlV6rUIIISqf9LyIGmfTpk2MGTOGL7/80nDsq6++YsyYMYSEhBiOtWvXDktLS5KTk4mMjDRDpUIIIe6GhBdRrRw7doyYmJjbtrly5QrLli3jn3/+MRwrbY0Xa2tr2rZtS9OmTWXGkRCiWmnYsGGxX9BUKhVr1qy5p+esjOcwFbltJKqVp59+muPHj7NmzRqGDx9eapubp0vr90DSb3l/89b0AHv37sXa2tqoNQshhLHFxcXh6uparrZvv/02a9asITQ09K6fw9wkvIhqIy0tzfCPrWvXrmW2CwgIQKPRkJSURExMDOnp6Wi1WpycnPDy8irWVoKLEMJc8vPzDbtI3ytvb+8q8RymIreNRJW0cePGYuNTQNdLoigKzZo1w8fHp8xrbWxsDGu4HD9+vNgto7IG5Wq1WmrYNl9CCBPr06cPU6ZMYcqUKTg7O1OnTh3efPNNw8+Whg0b8t577zF+/HicnJyYPHkyAHv27KFXr17Y2tri5+fHCy+8QFZWluF5ExMTGTZsGLa2tjRq1IjFixeXeO1bb/lER0czZswY3NzcsLe3p3Pnzhw8eJAFCxbwzjvvcOLECVQqFSqVyrCZ7a3PcerUKfr27YutrS3u7u5MnjyZzMxMw/mJEycyYsQIPvvsM3x8fHB3d+f555+noKCgEt/V0kl4EVVOcnIyQ4cOZdCgQbzzzjuGf/g7d+4ESl/f5VYdO3YEdGNkbl5ZtzRDhw7FxcXF0E4IUbUoikJ2fqFZHhX9pWbhwoVYWFhw6NAhvvrqK7744gt+/vlnw/nPPvuMdu3acfz4cd58800uX77MwIEDGTVqFCdPnmT58uXs2bOHKVOmGK6ZOHEiUVFRbN++nT/++IO5c+eSmJhYZg2ZmZn07t2bmJgY/vrrL06cOMGrr76KVqtl9OjRTJ8+ndatWxMXF0dcXByjR48u8RxZWVkEBwfj6urK4cOHWblyJVu2bClWF8D27du5fPky27dvZ+HChSxYsMAQhoxJbhuJKmf//v0UFhYCunuzycnJfPPNN7ddnO5WHTp0YNGiRRw/fhw3Nzeg7A0ZU1NTycjIYN26dWUGHCGE+eQUFNFq1sY7NzSCs+8GY2dV/o9KPz8/5syZg0qlokWLFpw6dYo5c+YwadIkAPr27cv06dMN7Z9++mnGjRvHSy+9BECzZs34+uuv6d27N99//z1Xr17ln3/+4dChQ4blHX755RdD73JplixZQlJSEocPHzb8/GvatKnhvIODAxYWFre9TbRkyRJyc3NZtGgR9vb2AHz77bcMGzaMjz/+2HAL3tXVlW+//RaNRoO/vz9Dhgxh69athu/XWKTnRVQ5+/btA3RhQ6PR0LNnT7KysgyLyZU3vFhYWFBQUMCsWbNYu3YtjzzySKltJ06cCMAnn3xSrKtWCCEqqlu3bsVuTwcFBXHx4kWKiooA6Ny5c7H2J06cYMGCBTg4OBgewcHBaLVaIiIiOHfuHBYWFnTq1Mlwjb+/Py4uLmXWEBoaSocOHQzB5W6cO3eOdu3aGYILQI8ePdBqtYZb8QCtW7dGo9EYvvbx8bltr1BlMWrPy65du/j00085evQocXFxrF69mhEjRtz2mh07djBt2jTOnDmDn58fM2fONHy4iNpBH15ee+01+vTpQ+PGjdm6dSuFhYX4+fnRoEGDOz5Hjx49yMzMNAzIbdiwYZltJ0yYwOzZs4mIiOD777/n5ZdfrpTvQwhROWwtNZx9N9hsr12Zbg4DoLvF88wzz/DCCy+UaFu/fv27up1ta2t71/VVlH67FT2VSoVWqzX66xq15yUrK4t27drx3Xfflat9REQEQ4YM4f777yc0NJSXXnqJp59+usQ+NaLmKioq4vTp0wB0796dxo0bA7qelKVLl/LOO++UayVcCwuLcs8ksrS0ZNasWQB8/PHHxQakCSHMT6VSYWdlYZZHRVfePnjwYLGvDxw4QLNmzYr1TtysY8eOnD17lqZNm5Z4WFlZ4e/vT2FhIUePHjVcExYWRmpqapk1tG3bltDQUK5fv17qeSsrK0NPUFlatmzJiRMnivVG7927F7VaXSVurxs1vAwaNIj333+fkSNHlqv9vHnzaNSoEZ9//jktW7ZkypQpPPzww8yZM8eYZYoqRKPREBcXx6FDh4qNUXFzc+PRRx+t8LYAFy9e5IMPPii2YF1pHnvsMZo2bUpycjLffvvtXdUuhBBXr15l2rRphIWFsXTpUr755htefPHFMtu/9tpr7Nu3jylTphAaGsrFixdZu3atYWBsixYtGDhwIM888wwHDx7k6NGjPP3007ftXRkzZgze3t6MGDGCvXv3Eh4ezp9//sn+/fsBXU90REQEoaGhJCcnk5eXV+I5xo0bh42NDRMmTOD06dNs376d//3vfzz++OMllpwwhyo15mX//v3079+/2LHg4GDDG16avLw80tPTiz1E9WZlZUWXLl3Ktbni7fzzzz80b96cmTNn8tlnn922rYWFBW+99RYAv/76q0m6PYUQNc/48ePJycmha9euPP/887z44ouGKdGladu2LTt37uTChQv06tWLDh06MGvWLHx9fQ1tfv31V3x9fenduzcPPfQQkydPxtPTs8zntLKyYtOmTXh6ejJ48GACAgL46KOPDL0/o0aNYuDAgdx///14eHiwdOnSEs9hZ2fHxo0buX79Ol26dOHhhx+mX79+VeaXuyo12yg+Pr5EovPy8iI9PZ2cnJxSk+aHH37IO++8Y6oSRTVyc3dvebo5x4wZQ2pqKhMmTLjn4CSEqJ0sLS358ssv+f7770uc06/0fasuXbqwadOmMp/T29ubdevWFTv2+OOPF/v61indDRo04I8//ij1+aytrUs9d+tzBAQEsG3btjLrKm1K9M1bFhhTtf8JPWPGDNLS0gyPqKgoc5ck7kH//v2ZPHkyCQkJ9/xc+m0CAOrWrXvH9hqNhilTpuDo6HjPry2EEMJ4qlR48fb2LvGhlZCQgJOTU5n396ytrXFycir2ENVTVFQUW7duZf78+Tg4ONzz893ci+fn51ehaxVF4dy5c/dcgxBCiMpXpcJLUFAQW7duLXZs8+bNBAUFmakiYUr6KdLt27cvMZ3wbq1du5aXX36ZcePGlfua+Ph4unTpQuvWrZk9e3aZK2xu37692AwAIUTttmPHDpPdNqntjBpeMjMzCQ0NNWympx/dfPXqVUB3y2f8+PGG9s8++yzh4eG8+uqrnD9/nrlz57JixQqmTp1qzDJFFaEPL927d6+053zwwQf59NNPy5ymWBpPT0+6du2Koii88cYbPPzww2RkZJRo99NPP9G9e/cy72MLIYQwDqOGlyNHjtChQwfD2INp06YZRlKDbvttfZABaNSoEevXr2fz5s20a9eOzz//nJ9//pngYPMsTiTuzoYNG0r9sL8TY4SXu6FWq5k7dy4//vgjVlZWrFq1isDAQHbs2EFMTIyhnUajIT8/n3fffdeM1QohRO2jUmrYVrrp6ek4OzuTlpYm41/M4Ouvv+bFF19k6NChrFmzptw9HllZWTg7O1NUVERkZCT169c3cqXlc+DAAUaNGkVsbCwAo0ePZtmyZYZzQUFBqNVqzp07V+beScaSkpKCvb09VlZWJn1dIYwtNzeXiIgIGjZsaNLVYoXx5eTkcOXKFRo1aoSNjU2xcxX5/K5SY15E9RcYGIiNjQ3r1q3j1VdfLfd1R44coaioiLp161Z4cO3NCou0ZOcXkpZTwLXMPBLSc4lPy+V6Vj5ZeYUUFGkrtEtst27dOHr0KD179gQgJiaG3Nxcw7mhQ4ei1WoNa8SYSlJSEn5+fvTt29ekryuEKeh/6cnPzzdzJaKy6f+fVuRWfmmk50VUiszMTObNm0f//v0JCwvj0UcfBeCHH3647QJNeuvXr2fatGm0a9eOFStWlDhfpFW4ej2bqOvZRKfkEJ2i+29iRi5pOYWkZeeTllNAVv7tl7wGUKvAydYSN3sr3O2tcLWzoo6jNX6udjRwt6O+mx313e1wsrmxZ0dRURGXL1+mWbNmxdaP0W+ABroN1tq2bXvH168My5YtY8yYMYDuva+sAc5CVAWKonD16lUKCgrw9fWVdZdqCK1WS2xsLJaWltSvX7/E1gsV+fyW8CIqxYYNGxgyZAiNGjUiPDyc9957j1mzZmFhYUFISAj9+vUr1/MUFhaiUms4E5vGieg0zsamczYunbD4dHILKr7qrUat+8dRpK34X3NvJxva1HUmoK4zAfWcCKjrgodjyf2SRo8ezYoVK3jwwQdZu3ZthV/nbixcuNCwYemhQ4fo0qWLSV5XCFPJz88nIiJCVruuYdRqNY0aNSr1dndFPr+r1Aq7ovrasmULgGF7h5kzZxIWFsbixYsZNWoUISEhdOvWrdRrFUXhclImey9dY++lZA6EXyM9t7BEO1tLDfXd7Kjraks9V1vqutji7WyDi50VzraWuNha4mxribWlGgu1Ggu1CvVN4SW/UEteYRF5hVrScgq4npVveCSm53L1ejaR//buJGfmE5+eS3x6LlvO3Vh7qJmnA/c196BXszoENnLH1krDO++8wx9//EFoaCgpKSm4urpW9ttbQnR0tOHPJ0+elPAiahwrKyuaNWsmt45qGCsrq0rpSZPwIiqFfn0efQ+LSqXi559/JiIign379nHmzJkS4SUiOYu/QmNZGxpNeHJ2sXOO1hZ0bOBKa18nWvk60crHiQbu9oaelIrSqFXYWmmwtdLdZ/Vysrlt+4zcAs7HZ3AqOo3TMWmciknjUlImFxN1j1/2RGBloaZHE3dGdKjL2vX/8MD9vcu9k/W9mjJlClu3bmX79u2cOnXKJK8phKmp1eoSgzqFALltJCpBQkIC3t7eACQmJuLh4WE4l5GRwbJly5g0aRIAWXmF/HE0mlXHojkRnWZopxTm46nOZOKgILo3cSegrjMWmqp1nzs1O599l6+x60ISuy4kEZuWazjnYG3BoDbePNSxHoGNXE1yj/7y5ctkZWXh7+8vM46EENWejHmR8GJSS5cuZezYsbRv357jx4+X2iYpI48F+yL4bf8V0nP/HVSraCmIOkXaya1kX9zPq1Nf4KOPPjJh5XdPf6vrrxNxrDoWTXRKjuGcTe41+vkW8eW08Vha3NuIeiGEqC1kzIswqVvHu9ws6no2c3dc5s9j0eQX6gbeFVyLJuPYOrLO70GbnYqTkxPDBj7Af//7X5PWfS9UKhVNPR2Z9oAjL/VrxpHIFFYdi2bV0avk2riz/jpseW0lrw/vyNjuTbCu5BDz2muv4e7uzjPPPIOzs3OlPrcQQlR10vMi7pm/vz9hYWGEhIQYVkPOyS/i+52X+WHnZfL+DS0d6rvwcGsXQn75hIz0NO677z569+5N+/bt73nOf1WRmp3Pc18sY0+SJRpb3d8/D3sN74xox6A23iWmBt6NnJwc7OzsAN2igKGhoUyePJnAwMB7fm4hhDAXuW0k4eWeZGVl8fLLL+Pn58drr712x2CRm5vLvn376NatG7a2toScjuf99eeISdXdSglq7M60Ac3p3MC1Uj68q4OtO/cw8d0fUFr0w8JJNwaoV7M6vPNgaxp73NuO2ZcuXaJZs2bY2dkxePBg/vjjDz777DOmT59eGaULIYRZyAq74p5ERUUxb948Zs+ezfXr1+/Y3sbGhr59+5Kar+LxXw7x3OJjxKTmUNfFlrnjOrJkUiBdGrrVmuAC0K93T44u/Rzvoz+QuncJKm0huy8mM/DL3Xy2MYycciymVxb9NOl69eoREBAA6KZLCyFEbSHhRZSg/3DMyspi9OjR5bpmy9kEBn+9mz2XkrGyUPNC36ZsmdabwQE+tSq03MzT05PFixaQe/hPkha9ROe6tuQXafl2+yUe/HYPlxIrvnkl6MIlgJ+fn2FFX5kuLYSoTWTArihB/+EIcOzYMRRFKTWAFBUVcV+f+7ENfJRLFg0ACKjrzDdjOtCwjixXD9CiRQsWLFhA586dadKkCRvPJPDm2tNcTMxk2Dd7mf1QG0Z2qFeh5yyt5+Xs2bMUFhZiYSH/pIUQNZ/0vIgSbl69NS0tjcuXL5fabuOeI4Q3GmEILk/0aMgfzwVJcLnFo48+StOmTVGpVAxs482GF3rRo6k7OQVFTF1+gtf/PEluQflvI90cXho1aoS9vT15eXlcvHjRWN+CEEJUKRJeRAk3hxfQ9b7c6nRMGtND4rD2bYGmKI8fHu/EW8NaV/qU4Jpm7969nDm6n0VPBvJS/2aoVLDscBQj5+4jLi3nzk9A8fCiVqtp06YNILeOhBC1h4QXUYL+tpGlpW5X5aNHjxY7fyD8GmN+PECOYkl+wmWerJtIcGtvk9dZ3axatYqePXsyfvx4srMyeal/c357MpA6Dlaci0vn4e/3E56Uecfn+fXXXwkNDWXkyJEAhnEvkZGRRq1fCCGqCgkvogT9b/YjRowAioeXTWfiGT//EBl5heRFnSZ+yQweHnS/OcqsdoKDg2nUqBFRUVF88MEHAPRsVoe1U3rSuI49Mak5PDJvP6dj0m77PG5ubrRr1w4vLy8A3n//fdLT03nllVeM/j0IIURVIOFFlLB7925Onz7N//73P+DGoN0VR6J49vej5BdqaeOqJX75mzT286F58+Zmrrh6sLe355NPPgFg+fLl6JdYqutiy4png2jt68S1rHzG/HiAg+HXyv28np6eODo6GqVmIYSoiiS8iBKcnZ1p3bo1gYGBDB8+nBdeeIHF+yN49Y+TaBV4pFM9PC/+BUUFDBkypNZOhb4bgwYNwtramitXrnD+/HnD8ToO1iyd3I2ujdzIyCtk/PxDbD+fWOL6uLg4XnjhBb766itTli2EEFWKhBdRJisrK9asWUOP/zzHrL/PAfBUz0Z88nBbfLw88fX1ZciQIWausnqxt7enT58+AGzYsKHYOScbSxY92ZX+LT3JK9Ty7O9HOXyl+CKBFy9e5JtvvuHbb78tdvztt9+md+/eHDlyxKj1CyFEVSDhRRRz4sQJnnvuOX766ScAjkZeZ8qSY2gV+E/neswc0hKVSsUHH3xAdHR0qZsxitsbPHgwUDK8ANhYavj+sU6GAPPUgsNcSLixmN3NM41udvDgQXbt2lVicLUQQtREEl5EMaGhocybN4+VK1dyKTGDpxYcIa9QSwcvS2aPDCh2i0ilUqFWy1+hitKHl/DwcAoKCkqct9So+WZMRzrWdyE9t5AJ8w8R++8+UWWFF/1idTJdWghRG8gnjyhG/+HoUb8pE+YfJjWngLzYMHZ9NAGNWhdcTp06RVHR3e/NU9s1bdqUU6dOER4ebpiOfitbKw3zJ3ahqacDcWm5TJh/iLTsgjLDi366tOxxJISoDSS8iGKio6NRWVhxzqMPMak5NHS3I2XNB1xPjCcyMpLExETatWuHr68vGRl3tzePgDZt2txxoLOLnRULn+yKt5MNFxMzeXrRYa7GxAG373mpYRvFCyFECRJeRDFRUVG4PfAs1xV73O2t+O2pQFo3awjo1nv5559/UBSFunXryvTcSqDVam8bNuq62LLwya442lhw+EoK5+10PSy3hhd/f38sLCxITU0tsUKyEELUNBJeRDGXFQ8c2g5ABXw9pgN+bnZ06tQJ0IWX9evXAzB06FAzVlkzPPvss3h5ed1xnEoLb0e+H9cJlQpy63bEPqB/ifBibW1NixYtABn3IoSo+SS8CIPz8elktxwGwNi2zvRoWgeAjh07AnDgwAE2btwIIFOkK0FMTAzJycmlzjq6Vc9mdZjWX7cYoO+wl1C7+ZVo07ZtW3x8fMjPz6/0WoUQoiqR8CIAyMwr5LnfjqKysCYn/CjTB7U2nNP3vGzfvp309HQ8PDzo0qWLuUqtMfSzjv75558S57Kzs0vcTnr+/qb0aeFBfhFM/eMM6bnFZyrNmzePmJgYw7YOQghRU0l4ESiKwoxVp4i4lo2noxWrXxuBq4uL4XxAQAAWFhaGrwcNGiRTpCvBoEGDAN1O06mpqYbj8+fPx93dnSeffLJYe7VaxZz/tKeuiy1XrmXz6sqTxQKOk5OTrHYshKgV5BNIsOxwFH+fiMVCreL7xzrRvVPbYh+CNjY2zJ492/C13DKqHA0bNqRly5YUFRWxefNmFEXh1Vdf5amnniI3N5fFixeTkpJiaL9nzx5mvT6d4R5JWGpUhJyJ55c9ESWeV6vVkpl5592phRCiupLwUstFp2Tz/rqzALwS3IJODdxKbffyyy+zZ88eZsyYwYABA0xZYo1282q7KpWKvLw8QNeLUlBQwOrVqw1tDx48yLfffsuJbX/x5tBWAHwccp7z8emGNj///DPe3t68+eabJvwuhBDCtCS81GL620VZ+UV0auCKS8IxnnvuOdatW1eirUqlokePHsyePRuXm24piXtz87gXrVbL559/zubNm3nttdcAWLZsmaFtVFQUoJsm/Xi3BvRv6UVBkcL0FScoKNICuk01k5KS2Lx5s4m/EyGEMB0JL7XQpk2b2Lp1KyuORLH7YjLWFmo+ebgtO3dsZ968eRw6dMjcJdYaPXv2JCgoiCeffJLc3FwsLCzo378/o0ePBmDbtm0kJup2l755dV2VSsXsh9rgYmfJmdh0vt9xGYC+ffuiUqk4c+YMsbGx5vmmhBDCyCzu3ETUJJGRkQQHB6NxdKfFS78DMH1Ac5p4OJS59LwwHisrK/bt21fieJMmTRg5ciRNmjRBq9X1qtz6/8fT0YZ3HmzNi8tC+XrrRfq19KS1rzudOnXiyJEjbNmyhfHjx5vumxFCCBORnpda5rfffgPAPfh/ZOUX0aG+C0/1bAwUvy0hzG/VqlV8+umneHt7A6VvyvhgO1+CW3tRqFV4eeVJ8gu1PPDAAwBy60gIUWNJeKlFFEVhwYIF2Lfph22TzlioFD59uK1hw0Xpeam6CgsLiYsrua+RSqXi/REBuNpZci4une+2XzKEly1btsg+R0KIGknCSy2yd+9eIuKu4dZvEgDTg1vS1FO3P1Fubi7JycmAhJeqpKCggI0bN/Lbb7+h1WqxsLDA09OzWBsPR2veG9EGgO+2X8KlURtsbW2Jj4/n9OnT5ihbCCGMSsa81CILFizAtfcE1DYOtPZ1YlKvRoZzMTExANjZ2eHq6mquEsUtfvzxR6ZMmUJQUBDXrl0jPj4ejUZTot3Qtr5sOBXHhlPxvLP+ApMmT8bO1hYnJyczVC2EEMYlPS+1SKcBD+PQVndL4d3hbSjIzyMyMhK4EV70M1lE1TBy5EhUKhX79+8nMzOTVq1aldn2rWGtcbC2IDQqle7jpvHhhx/SoEEDE1YrhBCmIeGllijSKmxIcADg4U71iA7dhZubG0888QQA9913H6mpqYaNF0XV4OvrS+/evQFYsWLFbdt6OdnwUv9mAHz0z3lSs2WDRiFEzSThpZZYfjiKUzFpONpY8NpAf1q1akVubi579uwxLCXv7OxMw4YNzVuoKEG/5ssrr7zCtm3bbtt2QveGtPByJCW7gNnrTrN+/XouXLhgijKFEMJkJLzUAqcuhPP26mMATHugOR6O1jRt2pTGjRtTUFDA9u3bzVyhuJ1Ro0YZ/rx27drbtrXUqHl3uG5H8BVHY3no6ZdYuHChUesTQghTk/BSC7y6aBf5WGKZncTj3XRjIFQqFcHBwQBs3LiR999/n2effZZjx46Zs1RRCg8PD3r06AHAww8/fMf2gY3dGdmhLqhUuD3wXzbJei9CiBpGwksNdyo6lXMF7gA81tIKC82N/+X68BISEsLq1av54YcfZEn5KiokJIRjx47Rq1evcrWfMdgfeys11r7NOV/gTlpampErFEII05HwUoMpisIrSw6ASk3ehb28PH54sfN9+/bFwsKCy5cvG3pc/Pz8zFGquAMHBwc6dOhQ7vaejjZMH+APgMt949m6e7+xShNCCJOT8FKDbT6bwPnrRWgL8ujjmoq9vX2x846OjobbEXqyQF3NMT6oAXYFaWhsnfh5X5S5yxFCiEoj4aWGKijS8lHIeQAyjqylb7fSf2t/7rnneO655wCwsbHBzc3NZDUK47LQqBnmVwjA2UJPoq5nm7kiIYSoHBJeaqhlh6MIT8pClZdJ2oE/qF+/fqntRo8ezaOPPgrobhnJAnU1y4QBncm5chzUFnz0zzlzlyOEEJVCwksNlJFbwFdbdGt75B5ZhZKfXWZ4AdlNuiZr2bIl/+tVDxWw/lQ8oVGp5i5JCCHumYSXGuiHneEkZ+bTuI494Vt+IzIykqZNm5bZPisrCwArKytTlShMRKVSMfP5iTzUURdMZ68/JztNCyGqPQkvNUx8Wi4/7wkH4NWB/thaW1G/fn0sLS3LvCYoKIju3bvzxhtvmKpMYWIvBzfH2kLNoSvX2XQ2wdzlCCHEPZFdpWuYzzeFkVugpXMDV4Jbe5XrmoCAAPbu3WvkyoS5ZGVlsXrxQnwzcomwbcFH/5ynr78nlhr53UUIUT3JT68a5Hx8On8ciwbgjSEtWbduHePGjZPl4Ws5jUbD1KlT2TlvJi42GiKSs1hy8Kq5yxJCiLsm4aUaysnJIS4ursTxzzaGoSgwOMCbDvVdOXToEEuWLOHgwYNmqFJUFTY2NnTp0gUlP4fuTqkAfLPtEtn5heYtTAgh7pLcNqrC5syZw4ULF6hfvz4NGjTAw8ODdevWsWjRIgYMGMDy5csNbY9GXmfLuUQ0ahXTB7QA4OpV3W/Xsmqu6NmzJ3v37iX37FbqNx/L1evZLNh3hf/2KXsgtxBCVFUSXqqwv/76ix07dpR6LjQ0lIsXL7Jnzx5cXFxZmqAb3/JIp3o08XAAbkyBvt00aVE79OzZk48//ph9e3bz4fNvMXX5CebtuMy4wAY425Y9mFsIIaoiCS9V2H//+1969epFZGQkV69eJSYmhjZt2jB58mQGDBjA4sWLefLJJwkc8QTxLUZhZaHmxf7NDNdLz4vQ6969OwBhYWEE+VrR3MuBCwmZ/LQrnJeDW5i5OiGEqBgJL1XYI488wiOPPFLm+WbNmgEq4jy6oAImdm+Ij7MtAFqtluho3eBd6XkRbm5utG7dmjNnznDwwH6mDwjkmd+OMn9vBBN7NKSOg7W5SxRCiHKTAbtVUGxsLB988EGxMS2ladasGXYte6Fyq4+DtYbnejcxnEtKSiIvLw+VSkXdunWNXbKoBnr06IFareby5csMaOVFu3rOZOcXMXf7ZXOXJoQQFSLhpQo6c+YMM2fO5L333rttOycXV9x7TwBgeHN7XO1vrJAbFxeHWq3Gx8fntgvUidrjnXfeITU1lWnTpqFSqQy3i34/EElMao6ZqxNCiPIzSXj57rvvaNiwITY2NgQGBnLo0KEy2y5YsACVSlXsYWNjY4oyq4yYmBiAO/aYrDgShdrZi6KsFFpZFF81tX379uTl5XHs2DGj1SmqF29vbxwdHQ1f92xah26N3cgv0vLN1otmrEwIISrG6OFl+fLlTJs2jbfeeotjx47Rrl07goODSUxMLPMaJycn4uLiDI/IyEhjl1mllCe85OQX8dUW3QdO2r7lXA0v+eFjYWGBl1f5VtkVtY9KpeKVf3tfVh6N5kpylpkrEkKI8jF6ePniiy+YNGkSTzzxBK1atWLevHnY2dkxf/78Mq9RqVR4e3sbHrXtA7g84WXh/iskZuThoMojIzSECxcumKo8UY0tXryY7t2789lnnwHQqYEbff09KdIqfL1Nel+EENWDUcNLfn4+R48epX///jdeUK2mf//+7N+/v8zrMjMzadCgAX5+fgwfPpwzZ84Ys8wq507hJS2ngO936AZZ/rdXAzaFbOCDDz4o1uatt95i3LhxsmeRKCY+Pp79+/cXW3X5pX+n1685HsPlpExzlSaEEOVm1PCSnJxMUVFRiZ4TLy8v4uPjS72mRYsWzJ8/n7Vr1/L777+j1Wrp3r27YdrvrfLy8khPTy/2qO5iY2OBssPLT7vCScspoLmXA88M7MgDDzyAr69vsTYbN25kyZIlt709J2qfgIAAAE6dOmU41raeC/1beqJVkLEvQohqocrNNgoKCmL8+PG0b9+e3r17s2rVKjw8PPjhhx9Kbf/hhx/i7OxseNSEBdlu1/OSmJHLL3siAJg+oAUatarU55AF6kRp9OHl4sWL5OTcmGH0Uv/mAPx1IpZLidL7IoSo2owaXurUqYNGoyEhofhMmISEBLy9vcv1HJaWlnTo0IFLly6Ven7GjBmkpaUZHvol8auzLVu2sHHjRlq0KLny6XfbLpFTUEQ7PxcGtNL1aK1atYo333zT8B7l5+cberZkgTpxM29vb9zd3dFqtZw7d85wvE1dZwa08kKrwNfS+yKEqOKMGl6srKzo1KkTW7duNRzTarVs3bqVoKCgcj1HUVERp06dwsfHp9Tz1tbWODk5FXtUd61atWLAgAHY29sXOx51PZslh3Q9Kq8Gt0Cl0vW6fPnll7z//vuGKegxMTEoioK1tTUeHh6mLV5UaSqVqtRbR3Cj9+Xvk7FcTMgweW1CCFFeRr9tNG3aNH766ScWLlzIuXPneO6558jKyuKJJ54AYPz48cyYMcPQ/t1332XTpk2Eh4dz7NgxHnvsMSIjI3n66aeNXWqVN2fLBQqKFHo0dadH0zqG482b6z509DOO9L1Pfn5+hoAjhF5Z4aWVrxMDW3ujKPCl9L4IIaowo+9tNHr0aJKSkpg1axbx8fG0b9+ekJAQwyDeq1evolbfyFApKSlMmjSJ+Ph4XF1d6dSpE/v27aNVq1bGLrVKCA0NZf369bRv354hQ4YYjl9IyGD1cd1YmFeC/Ytdo9vjSDeOAWS8i7i9gIAA/Pz8sLOzK3Huxf7NCDkTz4ZTcYTFZ9DC27GUZxBCCPMyycaMU6ZMYcqUKaWe27FjR7Gv58yZw5w5c0xQVdW0e/duZs6cyahRo4qFl4//OY+iQHBrL9r7uRS75tbwkpKSglqtlvEuolRPP/00kyZNKvVcSx8nhgT4sP5UHF9tvcDccZ1MXJ0QQtxZlZttVNuVNtPoYPg1tp5PRKNW8epA/xLX3HzbSFEU/ve//5GXl8fXX39tmqJFtXKnW4kv9NOF4Q2n4gmL1419+eOPP5g2bRp5eXlGr08IIe5EwksVc2t4URSFD/85D8CjXfxo4uFQ4pomTXS7SaelpZGcnAzotgaoCYOXhXFptdoSx1p4OzIkQDdA/uttF9FqtTzyyCPMmTPH0LsnhBDmJOGlitGHF/2ic/+cjic0KhU7Kw0v/rsS6q1sbW0Nt4jkw0WUx6xZs/Dy8uK7774r9fz/+jUFYMOpODbsPQ6Avb19qdP3hRDC1CS8VDE397wUFGn5dGMYAE/3aoynY9m7a69cuZJLly7RtWtXBg4cyNixY0lKSjJJzaL6KSoqIjExscSMIz1/bycGtdHNPPp2m24riu7du2NpaWnKMoUQolQSXqoQRVGKhZdlh6OISM6ijoMVk+9rfNtru3btSpMmTcjOzmbjxo0sXboUG5uyw46o3dq0aQPA6dOny2yjH/tyPtsOS3c/7rvvPi5cuMDq1atNUqMQQpRFwksVkp6eTlZWFgAuHl58tUV3C+iFfs1wsC7fxDD9Gi8uLi44Oso0V1E6/Vovp0+fRlGUUtu09HFiYGsvUKlw7v4odnZ2tGjRgokTJxr+ngohhDlIeDGx/Px8XnnlFcaPH09hYWGxc/b29pw+fZrNmzez5GgCyZl5NHS3Y0zXO095jo2N5e233+axxx4DZFsAcXstWrTA0tKSjIwMIiMjy2w3opmu986uZS8GPDyexo0bk56ezooVK0xVqhBClCDhxcRef/11PvvsM3777Tf2799f7JyFhQWtW7emTZee/LAzHNAtSGepufP/pszMTN555x1CQ0MBWaBO3J6lpSX+/rpp92WNewGIPn2A7Av7UanU/Lg3ismTJwOUuVFqRWm1WkaMGMGTTz5ZKc8nhKgdJLyY2Ouvv274863hRe/jkPPkFBTRuYErgwPKt4Flo0aN0Gg0hq+l50XcSVnbBNxs8ODBTB2gCzl/n4yl97DRWFhYcPDgQU6cOHHH19iwYQNnzpwp83xYWBhr167l119/JTs7u4LfgRCitpLwYgKXL182/NnT05OPP/4YKBleQkJCeOGdL1h9PAaVCt4a1rrcexNZWlrSqFEjw9fS8yLupHv37vTp06fYgoi38vX1Zcazj/FAKy8UBZaeTGHEiBEA/PTTT7d9/sOHDzNkyBACAwPLbJOammr48627zwshRFkkvBjZvn378Pf3Z8GCBYZjPXr0AHTh5ebBkn/9/Tcr/805D3esR0A95wq9ln6bAJCeF3Fnzz//PNu3b2fChAl3bPvivzOP/j4Ry4OP6bYW+O23327bWxIWppvm37FjxzLb3BxY4uPjy1W3EEJIeDGyv//+m8LCwmK/pXbq1AlLS0sSEhK4cuWK4fjpDFusfVtgqdLyysCKLwam3ybgpZde4pFHHrnn2kXttnPnTr788kvOnTtHm7rO9G/phVaBIzkeNG7cGLVafdtbTufP61aGbtmyZZlt9OHF19eXtm3bVu43IISosSS8GFliYiKgGzugZ2NjQ4cOHQA4ePAgAFl5hUS7dwZgoJ9y2wXpyqLveQkPD8fKyuqe6ha1R3p6eqk9KIsXL2bq1KnMnz8fgJf63+h9mfvbn8TGxt72ltC5c+eA24eXunXrMnz4cF544QXs7e3v5dsQQtQiEl6MTL/KraenZ7Hj33zzDefPn2f06NEAzN1xCa21IwUpsYzt5HNXr6UPLzePsRHidh5++GGcnZ1Zs2ZNiXO7du0CoHfv3gD/9r54olUgJEq3LcXt6MPL1KlT2bNnT6lthg4dypo1a3jttdfu4bsQQtQ2El6MTN/zcmt46dq1Ky1atEClUhF1PZufdkcAkLJ9Pg3r17ur1+rRowdhYWEcP3783ooWtYaHhwdQcsZRfHw8YWFhqFQqwxgtgBf76W5Nrg2NITwps8znLSgoKLbP1u0C9cmTJ/niiy9k5V4hRLlJeDEyfXjRf0jcSlEUZq45TX6hlpwroeSHHy4RdMrL3t6e5s2by/4zotz006VPnjxZ7Pju3bsBaNu2La6urjfa17vR+/Lk5yto1aoV69atK/G8ly5dKrYIY1mDcbOystizZw/Tp0/nt99+u+fvRwhRO0h4MbKybhsBLFy4kAeensHOC0lYqOH65u/x8fEptl6LEMZU1lov+ltG9913X4lr9L0vVxQPLsanldrTl5aWVmysS1xcXKmv36ZNG55//nlAZhsJIcpPwosRFRYW0r17dzp06FBqeFm9fiNhdroN8p7r3ZgTuzfxxx9/mLpMUYvpN2iMioritddeIzY2Frh9eAmo50w/f09QqXEOGl2i1wagW7dunD17li+++AIoPbwoilJsqrSs8yKEKC8JL0ZkYWHBxo0bOXbsGE5OTiXOF7QagsbeFeu8FKb0a06rVq1uO3tDiMrm6urKwIEDAfjkk0949NFHyczMNKzRUlp4AXjx35lH9q37EHo5tszn9/X1BUrvVcnIyCAnJ8fwdXx8fJmbRAohxM0kvJjJoYjrnMl1ASB96zysyrF/kRDGsH79ev766y969uzJ1KlTcXBw4Pr16+zdu7fM8Vdt67nQq7EzKrWG1LpBJXaZ1ocQb2/d9hal9bzoA41+Fens7GwyM8seBCyEEHryiWlEZf0WmVdYxP+t1o0xyDq1mfhTe3n77bd57733Su2CF8KY1Go1w4YNY/fu3Yal/+3s7Ojevfttr3ttiG68jF3L+9h04MY+R1qtFm9vb7p06YKFhQVWVlalrjukDy9NmzY1rPEit46EEOUh4cWIFi1ahIuLS4kdc+ftCOdSYiZ1HKxolBYKwLvvvsusWbPKtdmdEMZS3r20QLfui33qZVRqDT/vjzEcj4yMJDExkZMnTxIYGEhubi6nT58ucb0+vHh7ext6aGTQrhCiPCS8GFFiYiJpaWnFpoyejknj2+269S9mDWtNjy4dil1zu03yhKhq7nPLAOBspi0XE/7989mzgG67CgsLizID0c3hZeHChRw+fNiw8rQQQtyOhBcjunWButyCIl5aHkpBkcKAVl4Ma+tDUFBQsanREl5EdfL97DcY2NobBRVfbtWFcv3Kuq1atbrttQ0aNGDEiBEEBQXRo0cPOnfuLFsECCHKRcKLEenXeNEvUPfRP+e5lJiJh6M1H41qi0ql4sEHHyQm5kaXu352hhDVhX7m0YZTcYTFZ5TY02jmzJl0796dkJCQYtcNHz6c1atXM3XqVNMWLISo9iS8GNHNPS87whJZsO8KAJ8+3BY3e90ARhsbG1JSUgBwdHTE0dHRLLUKcbda+jgxqI0XigJfbb1Qoufl/Pnz7N+/v9h2Abc6deoUn3/+OX/++adJahZCVG8SXoxIH15sXDx45Q/dLKKJ3RvSp0Xx6af6nhe5ZSSqo7Fjx/Lba2NQobDhVDznE3TTnfU9Lz4+uo1Gb50unZmZaZiRt2/fPl5++WUWLVpkwsqFENWVhBcj0t82WnXVmqSMPJp6OvD6IP8S7fTTo8+fP2/S+oSoDBqNhqzYSzSySAWg3sBnaN68uWGXc314uXUmUfPmzbG1teXkyZMy20gIUSESXoyoY8eOtHrwGQ7H5WOpUfHl6PbYWJbct2jChAlMmDCB7du3m6FKIe5Nu3btALCP2IFaBemODVm+5SA2NjZA6QvVabVaEhMTycvLw93dHS8vL0DWeRFClI+EFyP6vzm/kNf6QQBeCW5Bm7rOpbZzc3NjwYIF9OnTx4TVCVE52rZtC8CFI7t5qGM9AD7fFGY4X9pto2vXrlFUVAToxoTd3PMiWwQIIe5Ewss9mD9/Pi1atODAgQMlzsWl5fDc78co1CoMCfBhUq/GZqhQCOPT97xcvHiR8R3rYKlRsftiMgfCrwGl3zbS/7lOnTpYWloael7y8vJIT083ZflCiGpIwstdWrVqFU899RQXLlxg48aNxc7lFhTx7G9HSc7Mw9/bkU8faVuhlUuFqE68vLzw9PREURR6tPcn6+RmQNf7oigK3t7eWFlZYW9vj1arBYovUAdga2tr2LxUxr0IIe5Ewstd2LNnD2PHjgXgmWeeYdasWYZziqLwxurTnIhOQ5ubgcX++dhZWZirVCFMQn/rKDMzk8Qdi7BUw+ErKey6mIyXlxe5ublcvnwZtVr3I+fW8ALIuBchRLlJeKmgc+fO8eCDD5KXl8eDDz7It99+W6xXZeG+K/x5LBoVCklrPoKsZDNWK4RpDBgwgPvvvx+AoszrjOmim/avH/tya89jaeFlwYIFHD58mE6dOpmiZCFENSbhpQJiY2MZOHAgKSkpdOvWjaVLl2JhcaNXJeR0HO+u0+3r0s0mjtzIE4atAYSoyV555RXef/99APz8/HjxgZbYWWk4GZ3GprMle1IaN27MyJEjCQwMNBzr3r27bBEghCgXCS8V8Oqrr3L16lWaNWvG33//ja2tLaNGjaJt27b8fegiLywNRavA6M5+eKfrVhnVbw0gRE1387YA7g7WPNmjEaDrfXnv/Q/o1q0bK1asAGDUqFGsWrWKKVOmmK1eIUT1JeGlAr777jtGjx5NSEgIderUQaVSceTIEcKu5fPy2gvkF2kZ1Mab2Q8FkHTLpoxC1HRnzpwBoH79+gBM6tUYJxsLLiRkciCukIMHD3LhwoUyrz916hSfffYZK1euNEm9QojqS8JLBTg7O7Ns2TIaN74x7ble6654PvIOeUXQq1kdvny0PRq1yrA1gPS8iNpizpw5ANjZ2QHgbGfJc32aAnDFsQ1oLAxrvaSnp5dYz+XAgQO88sorskWAEOKOJLzcg4jkLJJaP4rGxgFPdSY/PN4JawvdCrr6rQGk50XUFitXrmTo0KG88847hmMTuzfEy8maLGxwbD/IEF4aN26MjY1NsS0xZIsAIUR5SXi5S6dj0nhk3n7yNTbkJ0bQNHZzsSnRAQEBdOrUSTZbFLXGww8/zN9//42Li4vhmK2Vhhf7NQfAOWg0sUnXyc/P59q1a+Tn51OnTh1DW5kqLYQoL1mA5C7su5zM5EVHycwrxMemiEPL3ySqY5tibaTrWwidRzrX4+tNZ4jHhXinloZbqhYWFri5uRna6XteEhISUBRFFnYUQpRJel4q6J9TcUycf5jMvEK6NXbjwwFeaLNTuXz5srlLE6JKstSomRyk2yJA26IfYZGxgK6nRb9oHdy4xZqfn09qaqrJ6xRCVB8SXirg9wOR/HfJMfKLtAxs7c2CJ7oS0KIp9vb2eHh4UFBQYO4ShaiSRndvTkHCZdRWtvx6IAa4cZtIz8bGxnDLSca9CCFuR8JLOYWcjmfmmtMoCowNrM934zpiY6nBzc2NjIwMTpw4gaWlJQB79+7FycmJfv36mblqIaoGezs7lrz+KAD7ky3ROHkUW11XT8a9CCHKQ8a8lFNff096NatDh/quTO3fzHA/vrT78omJiWRkZJCTk2PqMoWosu5r7kFQY3f2h1/DpdfjeHullmizYMECLC0t8ff3N32BQohqQ8JLOVlZqPl1YhcsNHfurEqUBeqEKEGlUvH6IH+Gf7cXh9b306BeUok23bp1M0NlQojqRm4bVUBZwWXRokW0a9eO119/HUAWqBOiFF988QXPPDKQ1g45oFJx0a6VuUsSQlRTEl4qQW5uLidPnuTUqVOALFAnRGliYmI4ePAgDdJOYKlRsftiMrsuFO99OXXqFJ9++inLly83U5VCiOpAwksl0G8XEB4eDkjPixCl8fHRTZeOOh/K+G4NAJi94RxF2hvbBBw+fJhXX32VhQsXmqVGIUT1IOGlEjRp0gSAiIgItFqtjHkRohT62UXr1q3j/XH3YW+l5nx8BquPxxjayGwjIUR5SHipBH5+flhYWJCXl0dsbCz+/v506tTJsLuuEOJGzwtAfsZ1nump6335fFMYuQVFgIQXIUT5SHipBBYWFjRooPtBHB4ezty5czly5Ag9e/Y0c2VCVB03r+tiZ2fH5D7NqetiS1xaLvP3RhRrk5CQgFarNUudQoiqT8JLJdGPe5FtAoQo3c09Ly4uLthYWfBysG7Txu+3X+ZaZp7hVmthYSEpKSlmqVMIUfVJeKkkbdq0oXXr1oZVdoUQxbm6uhr+bGNjA8DwdnVpU9eJjLxC5my5gJWVlWGzRtkiQAhRFgkvleSLL77g9OnTdOzYEScnJzp37mzukoSoUlQqFd9++y0A7dq1A0CtVjFziG69lyUHr3IhIUPGvQgh7khW2K1k+q0BMjMzzV2KEFWOPpDcPP6lW2N3glt7sfFMArM3nOPXX3/F0tKSFi1amKtMIUQVJz0vlUzWeBGibC1btmTUqFF07dq12PHXB7XEUqNiR1gSua6N6dixI/b29maqUghR1UnPSyXJy8sjMDCQEydOALLGixClGTNmDGPGjClxvFEde8YHNeSXPRF8sP4sPZr0Ktc+YkKI2kl+OlQSa2trYmJuLLYl4UWIinmhbzNc7Cy5kJDJU7Pn8+uvv5q7JCFEFSXhpRLpp0uD3DYSoqKc7Sx5qV8zALYl2vLD/EVmrkgIUVVJeKlE+m0CQHpehLgb47o1wNdBg8behRjnNuYuRwhRRZkkvHz33Xc0bNgQGxsbAgMDOXTo0G3br1y5En9/f2xsbAgICGDDhg2mKPOe3dzz0qhRIzNWIkT1ZKlR83J/3b8jdct+XIiVheqEMDdFUe7cyMSMHl6WL1/OtGnTeOuttzh27Bjt2rUjODjYMCvnVvv27WPMmDE89dRTHD9+nBEjRjBixAhOnz5t7FLvmb7nZcCAAQwZMsTM1QhRPY3o2pT8yFBUGkveXnPC3OUIUav93//9H35+foSFhZm7lGJUipEjVWBgIF26dDEsTqXVavHz8+N///sfr7/+eon2o0ePJisri3Xr1hmOdevWjfbt2zNv3rw7vl56ejrOzs6kpaXh5ORUed9IOezatYvevXvTpEkTLl26ZNLXFqIm8e/ah5w+01CpNfz+VCA9m9Uxd0lC1EoqlQrAJJ9rFfn8NmrPS35+PkePHqV///43XlCtpn///uzfv7/Ua/bv31+sPUBwcHCZ7fPy8khPTy/2MJcmTZrQsmVL2rZtWyW72YSoLhq525JxbD0A7647Q2FRyU0aCwsLy+zBFUJUjm+++QbQ7dt39uxZM1dzg1HDS3JyMkVFRYblvvW8vLzK3LckPj6+Qu0//PBDnJ2dDQ8/P7/KKf4u1K1bl7Nnz7Jq1SpDWhVCVFzDhg1J27sEKwq4kJDJkkNXS7Q5cuQIXl5e+Pv7M2nSJBYuXEhsbKwZqhWi5poyZQoPPfQQAO+9956Zq7mh2s82mjFjBmlpaYZHVFSUuUsSQtyjadOmcXT/bl4O9gfgi80XSM3OL9ZGPw4uLCyMn3/+mYkTJ9KkSROio6NNXq8Q1ZFWW7JHszSzZs0CdGNYq0rvi1HDS506ddBoNCU2WEtISCi2t8nNvL29K9Te2toaJyenYg8hRPXWrFkz2rdvz5P3NaeFlyOp2QV8ueVisTZPP/00165d46+//uKVV17B2dmZ3NzcKvPDVYiqLDc3FxsbGxo0aFDmcIsLFy7w888/k5WVxYgRI1AUhffff9/ElZbOqOHFysqKTp06sXXrVsMxrVbL1q1bCQoKKvWaoKCgYu0BNm/eXGZ7IUTNZaFRM2uYbtfp3w5EEhafAUBcXByKouDm5sawYcP45JNPaN++PQDXrl0zV7lCVBsxMTEUFBSQlJSEo6NjqW127NjBpEmT+PDDD5k1axYuLi60bNmySozpNPpto2nTpvHTTz+xcOFCzp07x3PPPUdWVhZPPPEEAOPHj2fGjBmG9i+++CIhISF8/vnnnD9/nrfffpsjR44wZcoUY5cqhKgi8vPz+fTTT5kyZQqBDV0Ibu1FkVbhnb/PkJubS7169XByciI5OdlwTZ06dbCxsSEnJ8eMlQtRPeiHWOTk5DBnzpxS2+i3vKlXrx4dOnQgJiaGN998s0qM6TT6xoyjR48mKSmJWbNmER8fT/v27QkJCTEMyr169Spq9Y0M1b17d5YsWcLMmTP5v//7P5o1a8aaNWto00ZW2xSitrCwsGDmzJnk5+fzyiuvMHNIK7aHJbHv8jUWbTuJVqtFrVbj7u5uuGbp0qVYWlqasWohqo+bx4cePny41Db68WN169YFwM7OzviFlZNJdpWeMmVKmT0nO3bsKHHskUce4ZFHHjFyVUKIqkqtVlO/fn0uXbrElStX6N27Ac/e15ivt11i7oEEVBbWtGzZsthvgBJchCi/mwe2X71acjbfzW3q1atnOKYoCps2bSIpKYnHHnvMuEXeRrWfbSSEqJkaNmwIQGRkJADP9WmKr7MNqflqnAIfwt/f34zVCVG93dzzUlZ4ufm2kd6lS5fIy8tj3Lhxxi3wDiS8CCGqpAYNGgBw5coVAGytNLwxRDd41ynwYXybBRRrv3fvXh588EGmT59u0jqFqI5uDi+xsbEUFBSUaHPrbSPQzQR88MEHzT7uRcKLEKJK0ocXfc8LwOAAbyyvR6C2tOaMZfNi7VNSUvj777/ZtWuXSesUojpq06aNYRavVqs19LLoZWZmkpaWBhTveakqJLwIIaqkW28b6SVu/A5FW8SJ62r2Xrox20g/ePfmGUhCiNJ9+OGH7Nu3j6ZNmwIlbx1ZWVmxdetWfv/99zKnUpuThBchRJV0620j0O1l9uyjD+KReg6At/86Q8G/+x7pw4us8yJE+dWvXx93d/cSC9VZWVnRt29fs49tKYvRd5U2NXPuKi2EqDxZWVlcvHiRBg0a4OrqWuxcWnYB93++g+tZ+cwc0pKnezXm2rVr1Kmj2306Ly8PKysrc5QtRJVXVFQEgEajoaCgoEIz9QqKtMzecI6uDd0YFOBTqXVVmV2lhRDibtnb29O+ffsSwQXA2c6SV4NbAPDVloskZuTi4uJiGER4/fp1k9YqRHWyc+dObGxs6NevX5nBZc+ePfz888+cPHnScCwxPZexPx3g171XeOWPk6Rk5Zd6rSlIeBFCVBuXL18mISEBRVH4T2c/2tZzJiOvkI//CUOj0RiCjtw6EqJsUVFRFBYWotFoymyzdOlSJk2axMqVKwE4cuU6Q7/Zw+ErKThaW/DFf9rham++3k0JL0KIKmvFihVMmTKFnTt3AvDss8/i7e3Nb7/9hlqt4p0HWwPw57Fojkam4O7ujo2NDRkZGeYsW4gqTT9Nul69epw7d47g4GCGDRtWrI1+mrSvb10W7I3g0R8PkJiRRzNPB9ZO6cGA1qVvlmwqJllhVwgh7sb69etZtGgRdevWpXfv3pw/fx7QrTUB0KG+K490qsfKo9G89ddpQk+cxM7WxpwlC1Hl6cOLn58farWaTZs24eDggKIohluvuqnTKnbn1mPf37qd2oe09eGTUW2xtzZ/dJCeFyFElXXzWi8ZGRmG3wZbtGhhaPPqQH8cbSw4HZPOmpOJZqlTiOrk5vBSv359QLeuS2pqqqFNdEwM7oNfZF+CCrUKZg5pybdjOlSJ4AISXoQQVdjNa71cuHABAE9PT9zc3AxtPBytmdpft2DdpxvPk5ptvkGEQlQHN4cXW1tbPDw8gBtrveTk5lHYaSwOAf3RqODrMR14uldjs6+qezMJL0KIKuvmtV70t4xK29NofFADWng5kpJdQL+XvmTu3LkmrVOI6kTfg+nn5wdg6H2JjIyksEjL/xYfxqFNX5SiQr4e04GhbX3NVmtZJLwIIaqsm3tezp3TLUxXWnix0Kh5+9/Bu8ku/uwIvWSyGoWoTgoLCwkODqZ79+4lwsuVyKtMW3GCLRfTUIoK0RxcwJAqGFxABuwKIaowPz8/VCoVOTk57N69Gyg9vAAENXGnpV0257LtOGPTutjgQyGEjoWFBcuWLSt2TB9e/o625GJMLBZqFc+1t6N1v2fMUWK5SHgRQlRZVlZW+Pj4EBsbS1BQEJ06daJnz55lth/VVMW7R3LJsvNmTWgMIztUvQ3lhKhqGjRogFe3EVxU6f69fPVoB4a0rdzVcyubhBchRJW2fft26tSpg6ur6x17Upr4uJO2by6ufSYye8N5+rf0wtGm/EufC1HT5ebmYmlpWWyBut4PTeD7ZH/yi7S82K9ZlQ8uIGNehBBVXPPmzXFzcyvXLSB3d3fSj6xBmxZPUkYe32yTsS9C3Ozjjz/GxsaGV199FYD4tFye+f0Y+UVaglt78WK/ZqxYsYKffvqJiIgIM1dbNgkvQogq7+rVq+zbt++Oexa5u7tDUSHXt/wAwPw9EVxKlNV2hdDTbw3g4OBAbkERz/x2hKSMPFp4OfL5f9qjVqv48ssvmTx5MkePHjV3uWWS8CKEqNJOnjxJgwYN6NGjB//9739v29bd3R0Abcxp+jRzo1Cr8PZfZ1EUxRSlClHl3TxN+s01pzkRnYaLnSWqvT/SpX0AkZGR/66uq9s+oKqS8CKEqNJiY2MNfy5rppGejY0NWVlZZGdn8+6IdlhZqNlzKZmNZ+KNXaYQ1YJ+gboka19WHo1GrYLvxnbk8omDnD9/noiICMO/OQkvQghxl/RrvUDxbQHKYmdnB0B9dzueua8xAO+tO0dOfpFR6hOiOomKikJt68SSC7reyGd6N6FH0zqG6dJHjhyhsLAQtVqNt7d5N1+8HQkvQogqTf9DFW6sCFpe/+3TFF9nG2JSc5i383JllyZEtZKWlkZGRgZu/Z8hNbeI5l4OvNRft8mpfjXrffv2AeDt7Y2FRdWdkCzhRQhRpdnZ2TF27Fh69uxJYGDgHdt/9tlnDBs2jI0bN2JrpeGNIa0AmLfzMlHXs41drhBVVnR0NHYtemDfqjcatYrPHmmHtYVuyrT+lwR9eKnKt4xAwosQohpYvHgxu3fvxtLyzmu2HD58mHXr1hn2Qhoc4E1QY3fyCrV8sP5cqdcoioJWq63UmoWoarIKVfgMmwrAc72b0Laei+GcPrwkJCQAULduXZPXVxESXoQQNYp+xtG1a9cAUKlUvDO8NRq1ipAz8ey5mFys/dy5c/H392f9+vUmr1UIU1p4OpdCjQ3+3o78r1/TYuf04aVOnTps27aNGTNmmKPEcpPwIoSoUW4NLwDNvRwZH6S7p//232coKNL1sqSmpnLp0iUuXLjAwoULTV+sECYScjqO9afisLjldpFe/fr1cXd3p2XLltx///106dLFTJWWT9UdjSOEEHehtPAC8FL/5vwVGsulxEwW7rvC070a07lzZy5f1g3k/euvv7h27ZrheiFqityCIt7/95bp5Psa0aauc4k2zZs3Jzk5ucTxqkp6XoQQNUqdOnWAkuHF2daSVwfqplp/teUiYZFxhuDSsGFDCgoKSuy2K0RNMH9vBNEpOWjyM5g5sgsrV64ss+1XX33Fjz/+eMfVrM1NwosQokbR95yU9lvkI538CKjrTEZeIe+sOgboFr578cUXAeTWkahxEjNymbtdF9LVJ/+iIDfLEPBvpSgK//d//8czzzxDSkqKKcusMAkvQogaRR9esrNLTotWq1W8/aBu6vS+BAUr76YEBgYyduxYLCwsOHz4MGfPnjVpvUIYw6FDh9ixYwdfbLpAZl4hbes5E7v/L6Ds9ZKmTZtm+Hfj6+trslrvhoQXIUSN0rFjR7KzswkLCyv1fKcGboxo7wuocO3/DIGB3fD09GTw4MGA9L6I6i8nJ4e+ffsyYPSTLDt0FYCX7qtHdnYWUPYaLqdOnTL82dbW1viF3gMZsCuEqFEsLCzuuDLoq8EtWH04HJu6Lcn1dgFg8uTJODs78+CDD5qgSiGM59ixY2RlZeE17GlQqRja1gd3JQ3QjQmzsbEp9TqNRlPq8apIel6EELVOesJVUvctB2Dp2Rwy8woZMmQIixYtokePHmauToh7c/DgQWybBmLTQLc56euD/IvtJl2WV155BYCRI0eapM57IeFFCFHjvPTSSwwbNoyLFy+Wet7a2poJ3fyw02aRmJHHd9svmbhCIYxn34GDuN7/FACTejWinqsd6enpAHh6epZ5Xf/+/bl48WK1mHUn4UUIUeNs2rSJdevWERUVVer5Ro0a8fWcz/lywn0A/LI7gshruvEAJ06cYPr06aSmppqqXCEqRKvVEhoaSn5+fqnnDyeCpZsvjpbwXB/dSrqurq5YWFgwdOjQ2z5306ZNsbKyqvSaK5uEFyFEjVPWQnW3eqCVFz2b1iG/SMvsDedQFIVx48bxxRdfsHbtWlOUKkSFtWvXjg4dOnDw4MES56Jj4ihs8QCg63VxsNaN/xo4cCCZmZlMmTLFpLUai4QXIUSNc7vwkpOTw44dO8jMzESlUvHm0FZo1Co2nklg/+Vr9OnTB6DM2UpCmFtAQAAAW7ZsKXFue0Qmlq4+2KgKeLpP82LnrK2tTVKfKUh4EULUOLcLL0eOHOH++++nVSvdei8tvB0ZF6jblO7ddWepW083oFE/wFGIqmT79u3s3LkTKBleCou0/Lxf9/d22qAA7Kxq7oRiCS9CiBrnduFF39XeuXNnw7Gp/ZvjbGvJ+fgM4uwaA5Q5XkYIc9q2bRuxsbGA7u+yfiAuwOrjMURey8bd3orHujUwV4kmIeFFCFHjlLW/EdwIL4GBgYZjrvZWTO3fDIBtSfaore2l50VUSXFxcYY/FxUVsWvXLkDX6/LNNt2suf71wFqjMkt9piLhRQhR4+h7XjIyMkqcO3DgAADdunUrdnxctwY083Qgs0DBuccYoqOjURTF+MUKUQH68KJfiFF/62hNaCxXr2dTlJ3Gd1MfNVt9piLhRQhR44wbN46cnBxWrVpV7HhsbCzR0dGo1Wo6depU7JylRs2bQ3XjYBw7DqXQrs4dZysJYWr68PL4448DuvCi63XRrWmUfvBPOrcPqFar5d6NmjuaRwhRa5W1/Ln+llGbNm1wcHAocf6+5h708/dk6/lEhr/1K87OzkatU4iK0oeXcePG4e3tTf/+/VkbGkvktWwstXlkHF9P4NQXzVyl8Ul4EULUGmXdMrrZG0NasvNCEkdi89gXkUrv5h6mKk+I2yoqKiIxMRGAVq1a0a9fP7RaheAvdeNeNBe3oxTkFRvPVVPJbSMhRI2TlZXFhAkTGDZsGEVFRYbjEydO5JtvvmHs2LFlXtvYw4EJ3RsC8P66sxQWaY1drhDlkpiYiFarRa1WG5b533EhkYuJmdhbabgUsgCgVoQX6XkRQtQ4VlZWLFq0CICUlBTD7KOWLVvSsmXLO15/n3sWi1WFXEzMZOmhqzwe1NCY5QpRLh4eHly6dImkpCQ0Gg35+fnMXnUYUHFfXTVnczOpW7cudevWNXepRic9L0KIGsfS0hInJyfgzlsElObg7u3EbPwRgC82XyAtu+C27cPDw3nggQf47bffKl6sEOVkYWFBkyZNDLc9j19N4VK6CqWokPQjfwG1o9cFJLwIIWqomxequ3z5MkFBQSxfvrxc1/r5+ZEZGoJldjIp2QV8va303an1lixZwpYtWwy9PUKYwsKDurWIss7uoFPLxhw8eJA33njDzFWZhoQXIUSNdHN4+frrrzlw4AALFiwo17X16tUDRUvR0RUALNx3hfCkzFLbKorC4sWLAd0MECGM5e+//+aNN95g27ZtXEnOIuR0PADph1aze/duunbtSseOHc1cpWlIeBFC1Ej68HL58mV++eUXAKZNm1aua+vVqwdA7NGt9G3hQaFW4YP150ptGxoayvnz5wHYsGEDo0aNutfShSjVP//8w+zZs9m+fTs/7wlHq0AHLysKkiPZvn07hYWF5i7RZCS8CCFqJH14+fjjj8nKyqJNmzb079+/XNfqBzzm5eXxXJA3FmoVW88nsudicom2+l6XHj16sHLlSlavXl3qyr5C3Kv4eF1Pi7NnXVYe0d0ymj60HQBpaWm8/fbb5irN5CS8CCFqJH140f/Anzp1KipV+fZ7sbKywsvLCwCLnGuGTe7eX3+WIu2NLQOKiopYunQpAC+//DJ+fn4oisKRI0cq7fsQQk+/QN0FxYe8Qi0BdZ3p0dTDMDh9/fr15izPpCS8CCFqpNmzZ/Prr78C4Onpedu1XUrj5+cH6HaXfql/M8Ou08sP39hteteuXcTGxuLi4sKgQYMMMz30K/kKUZni4uJQWVizL9kSgMn3NUalUrFz504GDRrEsmXLzFyh6Uh4EULUSPb29sydOxeA//73v2VuGVCWuXPncuzYMfr164eLnRUv9tPtOv3F5jAycnVTpy0sLLj//vv5z3/+g7W1tYQXYTSKohAXF4d96/vJzFeo52rLoDbeALRv354NGzbQokULM1dpOhJehBA11owZM+jXrx/PPfdcha/t0qULHTp0wN7eHoDHgxrQuI49yZn5fLf9MgC9evVi27ZtfP/99wDFwovsSC0qU0pKCvn5BTh1GQ7AxO4NsdDU3o/w2vudCyFqNJVKxciRI9myZYthKfV7YalR88YQ3eq88/dEcPVatuGcWq37UdqpUyc0Gg1xcXFER0ff82sKoRcXF4dNow5YuvvhYG3B6C5+5i7JrCS8CCFEKSIjI/n000/5+uuvDcf6+nvSs2kd8ou0vPjrdhISEopdY2dnR9u2bWnevLlhoLAQlaF58+YMmfY5AP/p7IejjaWZKzIvCS9CCFGKqKgoXn31Vb766ivDMZVKxcyhLVGr4HgyNOzSn8uXLxe7bv/+/YSFhdGlSxdTlyxqsIjruRyJzkKtgid6NDR3OWYn4UUIIUqhX6guOjq62PgVf28nOjjnAOAz+HkaNmxU7Dpra2vTFSlqjV/3RgAwoJU3fm52Zq7G/CS8CCFEKXx9fVGpVOTn55OUlFTsXFHoWrR52Whd/FgdGlvq9UVFRWi1WlOUKmq461n5rDx8FYAuzqVvU1HbGDW8XL9+nXHjxuHk5ISLiwtPPfUUmZm3f+P79OmDSqUq9nj22WeNWaYQQpRw80J1Nw++1Wq17NnyD2n7dGtqfBJynqy84suyjxw5EhcXF06ePGm6gkWNtfhAJIWKiry4i6RdOmbucqoEo4aXcePGcebMGTZv3sy6devYtWsXkydPvuN1kyZNIi4uzvD45JNPjFmmEEKU6uaF6vROnjzJtWvX0J7fhp+rLYkZeczbWXzcS2ZmJpmZmbLei7hn+YVaFh2IBCDjyFp8fX3MXFHVYLTwcu7cOUJCQvj5558JDAykZ8+efPPNNyxbtozY2NK7WfXs7Ozw9vY2PPRLHwshhCndPO5Fb9u2bQDc17M7bwxpBcCPu8KJTrkxdVoWqxN3q6ioqNjX607GkpSRBzmpZJ3fg4+PhBcwYnjZv38/Li4udO7c2XCsf//+qNXqO/6DXrx4MXXq1KFNmzbMmDGD7OzsMtvm5eWRnp5e7CGEEJWhtJ6XrVu3AtCvXz+CW3vRrbEbeYVaPg4JM7SR8CLuxv79+/H09GT8+PGAblXdX/boBupmnwgBbaGEl38ZLbzEx8eXWBjKwsICNze3265/MHbsWH7//Xe2b9/OjBkz+O2333jsscfKbP/hhx/i7OxseOh/2AghxL164YUXOH78ODNmzDAcmz9/PsuWLWPkyJGoVCreHNoKlQr+PhHL0cjrwI3wcu7cOfmFSpSbt7c3169f57fffiMzM5P94dc4E5uOraWaawfXGNqIuwgvr7/+eokBtbc+zp8/f9cFTZ48meDgYAICAhg3bhyLFi1i9erVJdZS0JsxYwZpaWmGx82/IQkhxL1o0qQJ7du3x9nZ2XDMy8uL0aNH06iRbop0a19nRnfW/dL07t9n0WoVPD09adiwIYqicPjwYbPULqoHrVZrmIrfsGFDwx5cu3fv5pfdul6XB5o6oc3NxM7ODkdHR7PVWpVUOLxMnz6dc+fO3fbRuHFjvL29SUxMLHZtYWEh169fr1By1P8Gc+nSpVLPW1tb4+TkVOwhhBCmNH1ACxysLTgRncaq4zGA3DoSd5aVlcXIkSP58ssvAd0iiPrdz9ds3cfW84moVNDTU7cRqI+PDyqVylzlVikWFb3Aw8MDDw+PO7YLCgoiNTWVo0eP0qlTJ0A30E2r1Rr+UZdHaGgogNznE0KYXE5ODt988w0xMTHMmTOH1157DVdXV5544oliP5M8HK2Z0rcpH/1zno9DzjOwjTdDhw4lOzu7UvZVEjXTxIkT+euvv9iyZQtjxozB29ubfv36MX/+fLbHqcEP+vl7MaJfW8LDw8nKyjJ3yVWGSjHi1qeDBg0iISGBefPmUVBQwBNPPEHnzp1ZsmQJADExMfTr149FixbRtWtXLl++zJIlSxg8eDDu7u6cPHmSqVOnUq9ePXbu3Fmu10xPT8fZ2Zm0tDTphRFC3JOCggKsra1RFIWIiAj8/f3Jy8vj/PnztGjRoljbvMIigufs4sq1bJ7t3YTXB/nf1Wt+9NFHLFu2jJCQEBnfUIPFxcVRr149tFotO3bsoHfv3gAkJCTg26g5dZ/7FbWlNcsndyOwsbuZqzWNinx+G3Wdl8WLF+Pv70+/fv0YPHgwPXv25McffzScLygoICwszDCbyMrKii1btjBgwAD8/f2ZPn06o0aN4u+//zZmmUIIUSpLS0tDgFixYgV5eXnUrVuX5s2bl2hrbaHhzaG6qdPz90RwJbnivyWHhIQwY8YMTpw4wYYNG+6teFGlLV26FK1WS1BQkCG4gG5MVZPgiagtralnV0TXRm5mrLLqqvBto4pwc3Mz9LKURj+gTc/Pz6/cPSxCCGEKfn5+xMXFsXDhQgD69u1b5riDvv6e3Nfcg10Xknh//Vl+Gt/Z8FukWn373xWTkpKYOHGi4evw8PBK+x5E1bNo0SIAHn/88WLHcwuK0PjfDwo81NoFlUrFzz//THh4OKNGjTIMw6jtZG8jIYS4Df1CdWfPngV067uURaVSMWtoSyzUKracS8Qj4D5cXV1vuzyE3tKlS0lISDB8LeGl5jp58iQnTpzA0tKS0aNHFzv314lYchRLfJxt+N+InoCu1+/DDz/k9OnT5ii3SpLwIoQQt6EPL3p9+/a9bfumno6MD2oIgH2vCaDWEBMTc8fXeeGFF1iyZAlvvfUWIOGlJvvtt98AGDp0KG5uN24LKYpimB49sXtDLDW6j+i4uDhAJq7czKi3jYQQorq7eeHLZs2alWshzBf7N2NtaAzX8MGx41Cio6Pp0qXLHa8bM2YM6enpjB07lgYNGtxT3aLqevLJJ1Gr1SV68XZeSCIsIQM7Kw3/6VSXQ4cOYWNjI+GlFNLzIoQQt3Fzz8udel30nG0teTlYNxvJpedYzkfGldouPz+fV199laSkJMMxJycnmjdvjrW19T1ULaqyli1b8vHHHzNgwIBix+du1y3GOqZrfb767CMCAwN5//33uXbtGiDh5WYSXoQQ4jYGDBhAaGgo8fHxzJ49u9zX/aezHy7aNNTW9oTE25TaJiQkhE8//ZRevXqh1Worq2RRDR2+cp1DV65jqVExqVdjwwyklStXArqZb+7utWPKdHlIeBFCiNtwc3OjXbt2eHl5FRufcCcatYoHXK+jKFoi8eJA+LUSbcLCdJs5dujQodhspF9//ZUnnnhCZl/WMJmZmYwfP54NGzaUCKtzt+tWkX+4Uz28nW0ICgoybBUAuj2NZHXdGyS8CCGEkXRs6E5maAgAs9aepqCo+AeWflBu06ZNix3fvHkzCxYs4NChQ6YpVJjEqlWr+O2333jppZeKBZEzsWlsD0tCrYJn7msCgI2NDT179jS0kQULi5PwIoQQRuLv7083uySsKeBCQiYL910pdl6/4Wzjxo2LHdd/LTOOahb9LKPHH3+8WHiZu0P392BoW18a1rE3HNcP6G3Xrp1hXRihI+FFCCGMpEOHDqxfvYJ3R3UEYM7mCySk5xrO68PJreGlSRPdb9/6cCOqv+joaLZu3QrAY489ZjgenpTJhlO6Ad3P9WlS7Bp9eLly5UqJ3rnaTsKLEEIY2SOd/OhQ34Ws/CLeX38OgMLCQiIjIwHpeakNtmzZgqIodOvWjUaNGhmOz9t5GUWBfv6etPQpvp9Px44dcXFxIS0tjWPHjpm65CpNwosQQhiRoiikp6fx5qDmqFXw94lYdl9MIjo6msLCQqysrKhbt26xa/Q9L5GRkRQWFpqjbFHJzp8/D1Bsef+Y1BxWHdMtYPjf+0v2rGg0Gr7//nt27dpF+/btTVJndSHhRQghjKhnz566LQLOHTasvPt/q0/h4OLOpk2b+PXXX0vse+Tr64uVlRWFhYVERUWZoWpR2fQzy27ejfynXeEUahW6NXajUwPXUq979NFH6dWrF1ZWViaps7qQ8CKEEEakn14dHR3Ny8Et8HW2Iep6Dj/ui+aBBx5g7NixJa5Rq9WGWwtXr141ab3CONLS0oAb4SUmNYclh3T/b58vpddF3J6EFyGEMCL9Cr3R0dE4WFvw/sg2APy8O5xT0WllXrdlyxZycnIMi5WJ6m3btm1kZGQY/n9+teUC+YVaujV2o2fTOmaurvqR8CKEEEakDy/6zRn7+nvxYDtftApM+nkHVyJLvy1Ur169YouUierPwcEBa2trLiVm8MfRaABeHegvi8/dBQkvQghhRDf3vOjNGtYKVX428bkWfLH+uLlKE2by2cYLaBUY0MqLjvVLH+sibk/CixBCGJF+JtHN4aWOgzXZe3ULloXEWHAlOavEdRcvXuSpp57imWeeMU2hwmh++ukn7r//fn755RdCo1IJOROPWoVh805RcRJehBDCiG69bQSQkpJC4qG/yblynPwihdf+PIlWqxS7Li8vj/nz57NixQqT1isq36FDh9ixYweRkVf5+B/dlOmHOtajuZejmSurviS8CCGEEdWrV4/hw4czbtw4ioqKAIiIiABAfWQZdlYaDkZcZ/7eiGLX6WcbpaamkpKSYtqiRaXST5PW1G3N/vBrWGnUvNS/mZmrqt4kvAghhBE5ODiwZs0avvvuOzQaDXBj2f9Gnk7MHNIKgE82hhEWn2G4zt7e3rAZn2wTUL3pwouKXanOADzWrQH1XO3MW1Q1J+FFCCFM7OY9jcZ09aOvvyf5hVpeWh5KfuGNnadlm4DqLyUlhcTEROxb9eZySiEO1hY8f3+TO18obkvCixBCGJmiKKSkpJCamgrcCCNNmjRBpVLx0agAXO0sOReXzpdbLhiukw0aq7+wsDDU1va4958M6DZfdHewNnNV1Z+EFyGEMLJnn30WNzc3vvvuOwBmzpxJSEiIYXVdT0cbPnwoANBt1HfkynVAel5qgrCwMFx6T0Bl60RTTwcm9Wp854vEHUl4EUIII/Py8gJuTJf28/MjODi42D43A9v48FDHumgVmLbiBJl5hYbwou+xEdVPeJoWh/YDAXh/RBusLORjtzLIuyiEEEZW2nTp0rz9YGvquthy9Xo2r/95kocffpjs7GxWrlxpijJFJSss0nJIaYpKpeahDr50a+xu7pJqDAkvQghhZDcvVBcfH88777zD8uXLS7RzsrHk6zHtsVCrWHcyjhXHE7C1tS3Xa3z++ef8+OOPlVq3KJ/c3Fy+//57kpKSih1fsO8K5+LScbGz5I1/Z5WJyiHhRQghjOzmLQJOnTrF22+/zbvvvltq204N3JgxuCUA768/x9HIO6/xcvnyZV5++WX++uuvyitalNucOXP473//S2BgoOFYbGoOX2zWDb6eMchfBulWMgkvQghhZPrwkpSUxLlz54Abg3FL82SPhgwJ8KFQq/D4vB3c98Bgdu/eXWb7EydOABAfH1+JVdcsOTk5PPPMMyxcuLDSn/uff/4BdIsPJicnoygKb/11huz8IpTESyx+/3+V/pq1nYQXIYQwMjc3N6ytdb9579mzB7h9eFGpVHz8cFsae9iTrVgR5t6dk6dOl9n+1KlTAAQEBFRi1TXLJ598wo8//sjq1asr/blvvrW3fPlyfj94lc1nE9CoIO7vOUReuVLpr1nbSXgRQggjU6lUTJgwgeeff56oqCjgxhouZXGwtmDeY53QKEXYNuzA31eUMtvqw0uLFi349ttvGTduXOUVXwPk5eXx008/ATBkyJBKf379dg+NGzfGq0VH3lt3FoAguwQKkiOLzSoTlUPCixBCmMAPP/zAt99+S15eHnD7nhe95l6OBLslA3Be3YCVR6JKbacPL56enkyfPp0lS5awbdu2Sqq8+vv999+JiYnB19eX8ePHV+pzK4pCdnY2AOs3bWHuiTzyC7X09ffENuoAAP7+/pX6mkLCixBCmIyiKIbVcssTXgCGtPEibb9uqvSMVafYfbH4jJbs7GwuXrwIwODBg3nmmWcA3UJ4ilJ2b01tUVRUxCeffALAtGnTyM7O5sCBA5X2/CqViujoaNLT0/nxSBrhSVl4O9nw2SPtuPDvhozS81L5JLwIIYQJ6INLeno6cGPX6Dtp1aoVqbsWkX1uJ4Vahed+P8bZ2HTD+bNnz6IoCh4eHnh5eTFjxgxsbW3Zv3+/YSBpbbZ69WouXLiAq6srPXv2xN3dneDgYMMO35Vl04U0Vh2PQQW0zTqKi62FYTdpCS+VT8KLEEKYwNy5c2nWrBlt27Zl8+bN5V6/pWHDhnh7e5G0fg4t3S3IzCvkiQWHiE3NAcDCwoKHHnqIIUOGoFKp8PHxYcqUKYCu90Wr1d7u6Ws0RVH48MMPAZgyZQqdO3fG0dGR9PR0QkNDK+11LiZk8OZa3YDqrIMr+Gn2a6xfv56EhAQAmjdvXmmvJXQkvAghhAn4+voCYGdnR//+/ct9nUqlIigoiHo+3oxvnEszTwcS0vN44tfDpOUU0L59e/78809+/fVXwzWvvvoqjo6OHD9+nP/973/k5ORU+vdTHVy7dg2VSoWtrS0vvPACGo2G++67D4AdO3aUaJ+Xl8fAgQNp3rw5AwYMYNmyZXd8jdlfzmXQx+vJzi8iqLE7QxppAPjmm2/o3r07bdq0wcnJqVK/LyHhRQghTOLmheoqavHixURFRTFm1HAWPNkVT0drwhIyePyXg6Rm55doX6dOHWbMmAHAtm3bsLGxubfiq6k6depw+PBhTpw4QZ06dQDo06cPUHp4WbVqFRs3buTixYts3ryZ2NjY2z5/WnYBv0c5U2jliLM6j2/HdmD8448DcOjQIbZu3WoYTC0ql4QXIYQwgZvDy/Hjxyt07c23mOq62LLoqa642VtxMjqNR+bu5lpmXolrXn/9ddauXcsnn3yCSqUCdAu1DR8+vNLHe5hbSkoKQ4cOZejQobz88sv8/PPP7N69m8zMTFQqFc2aNTO0vf/++wHYtWtXiffhhx9+AGDSpEn8+uuvDByo21CxoKCgROjMLSji6UWHybZ0oTDjGs+30uLuYE3v3r3x8/MjLS2Nv//+25jfdu2m1DBpaWkKoKSlpZm7FCGEMCgsLFQABVA++uiju3oOrVarFBQUKIqiKGHx6UqHd0KUBq+tU+pN+l6JTr7zz7wXX3xRAZRz587d1etXVT///LPhvdU/OnXqpCQnJ5doW1hYqDg7OyuAcuTIEcPxc+fOKYCiVquVqKgow/EDBw4oVlZWSpMmTQzHCgqLlKcXHlYavLZOqT91hWLp0VA5fPiw4fyMGTMUQBk2bJiRvuOaqSKf39LzIoQQJqDRaAx/btu2bYWvf/311/Hw8GDp0qWAbg2Y6e3VFGZcQ+Pmx/gFx0hIz73tc+jXfgkPD6/w61dl58+fB3S3hF566SUGDhzItWvX6NevH1lZWcXaljXuRb+p5ZAhQwy9ZKCb0p6fn8/ly5fJzMykSKvwxurTbD6bgJVGTcLKdyhIulJs6vvj/946+vvvv9m/f79RvufaTsKLEEKYyPbt25kzZ47hdkRFFBYWcu3aNfbt22c4lhR+hoQlr2NZkMnlpCwenrePsPiMMp+jadOmAIa1ZmoK/ZTkhx9+mDlz5vDPP/8QERFBaGgo9vb2Jdo/99xz/PTTTzzyyCOGY4mJiahUKsM6OXoeHh54e3sDcPzkaZ5ffIzlR6JQq+DVXnXIiz6Dk5MTrq6uhmtatmzJyJEjqV+/Pi1btjTGt1zrSXgRQggT0fcM6MegVET37t0BioWXU6dOUZgax4P2l6jvZkfU9RwemruXTWdK36BR3ztQ03pevv76a9atW8fQoUPL1X7QoEE8/fTT1K9f33Ds999/Jzw8vNRg2bZtW9Q2DszYFEvImXisNGq+GdMRn6JEQPe+3vr/9M8//yQyMhIXF5e7/8ZEmSS8CCFENRAUFAToAot+oTv9TJagti1Y83wPghq7k5VfxOTfjvL11oslVtjV76dU03peGjZsyJAhQ2jQoME9P8/Nt/f0Ggd0wfuxT4nOtcbJxoJFT3VlSFsfMjIycHd3L3XBwbsJqKL8JLwIIUQ14OPjQ6NGjVAUhYMHD1JUVMSZM2cA3W7SbvZWLHqqKxOCdB/gX2y+wPNLjpGVV2h4jpra83I3wsPD+eabb/jhhx9uO339+NUUdtsEYunuhyYvnT+e6063xu4APProoyQnJ7N8+XJTlS3+JeFFCCGqiZtvHYWHh5OdnY2NjY1hLIulRs07w9vw0UMBWGpUbDgVz8CvdrH/8jXgRs9LeHh4jdn36NixY7z77rts3ry5QtetWbOGF154gWeffZYGDRoYVuLVKyzS8tWWizw8bz+ZhWryEyNIXTmTZp4OJZ7L0tLynr4HUXESXoQQopq4ObzY2Njw6quvMmnSpBK3Oh7tWp+lk7pR18WWqOs5jPnpALPWnsbdy5fnn3+eDz74gIKCAnN8C5Vu27ZtvPXWW/zyyy8Vuk6/WB2AVqulffv2hq8jr2Xxnx/2M2fLBYq0CoPbeBGUfYBpzz1ZY9636s7C3AUIIYQon549e3Lfffdx33334efnx8cff1xm284N3dg49T5mbzjHkoNXWbQ/ku1hiXw89W26N6ljwqqN6243P2zXrh3Ozs6kpaXRoEEDBgwYgFarsPJoFO/+fZas/CIcrS14f2QbhrevC4/9Xux6RVHo2LEjXl5eLFq0CE9Pz0r7nsSdSXgRQohqom3btuzcubPc7R2sLZg9MoDBbXx47c+TRF3PYexPB+nf0pNXgv1p4e1oxGpN427Di0ajoV+/fqxatYrJkydz8EoKszec43SMbjB010ZufPGfdtRztSv1+uTkZMPmjrJ3kemplJpy4/Nf6enphjQtf6GEEDXVvn37aNasGR4eHuVqn5lXyMf/nGfJwUiKFFABozrVY+oDzanrUr4drqsiLy8vEhMTOXLkCJ06darQtTExMSxcs4mLdq3YHpYM6ALfC/2a8lTPxmjUN2YMabVarly5QkpKCp06deLQoUMEBgZSt27du9qvSpRUkc9v6XkRQohqJi4ujh49egAQHx+Pl5fXHa9xsLbgvRFtKDiziV+OJGPfogd/HI3mr9BYRnWqy/ighrT0qV6/8KWmppKYqFtrpXnz5hW69vjVFBbsS+TvaE+0SjIatYpxgfV5oV8z6jhYl2i/YcMGhg0bRkBAACdPnjTM2CptmrQwPgkvQghRjezcudMw2LROnToVHmvRqbkfH8+cToMBo2g6ajoHwq+z9FAUSw9F0bWRGxOCGjKgtReWmqo/n0N/y8jX1xdHxzvfAssv1LLhVBy/7rvCiahUw/EHWnnx+iB/mniUnEmkFxAQAMC5c+fIz883hJebtwUQpiPhRQghqpGb90Xy8fGp8GJo+g/b6BN7OByykkMR11m0P5KQM/EcirjOoYjreDhaM6iNN4Pa+NC1kVux2ydVSXnGuxQWaTkYcZ31p+LYeDqea1n5AFhp1Axr58vE7g0JqOd8x9eqX78+Tk5OpKenExYWRkREBCDhxVwkvAghRDVy8x46+fn5Fb5ev9ZLQkICWVlZBDZ2J7CxO3FpOSw5eJWlh66SlJHHov2RLNofSR0HKx5o5U2fFh4ENnLDxc6q0r6Xe/Xoo4/SpUuXEu9DSlY+ByOusyMskY1n4knJvjG92dPRmse7NWBMYP1Sbw+VRaVS0bZtW/bs2SO3jaoACS9CCFHNvP3227z99tt8/fXXFb7WxcUFV1dXUlJSiIiIMNwO8XG2ZfqAFvyvbzP2Xkrmn9NxbDqbQHJmPksP6UINgL+3I4GN3OjSyI1WPk40cLc3W8+MlZUVzVv4c+VaFutOxnIo4joHw68TllB8c0o3eyuCW3sxqI0PQU3c7/qWWEBAAHv27OHUqVM4Ojri5uYmPS9mIuFFCCGqmZkzZ/Lss8+Wa6BuaRo3bszRo0cJDw83hBe9/Nxs/J2LuP/hdnxQpOVA+DU2nonnQPh1LiVmcj4+g/PxGSzcHwmAjaWa5l6OtPBypJGHPXVdbKnnaktdFzs8Ha1RV0KwKdIqJGbkEpOSQ0yq7hGRlMX5+AwuJGSQV6gtcU0zTweCmrgT3NqbwEZuWFTCGB79LbuTJ0+yYcMGgBqzUnF1I+FFCCGqGY1Gc9fBBXS3jo4ePVrqBo0DBw7k6NGj7N27l44dO9KrmQe9mummYydn5v3bu3GN41GpXEjIILdAy8noNE5Gp5WsU63C2dYSF1tLnO10/7WzssBSo8JSo8bSQo2FWkWhVqGgUEtBkZaCIoWs/ELScgpIyy4gNaeAtJwCirRlhwQbCzXNvR3p4OdCYGN3ujZyq9AtofLSB72TJ08ajskGjOYh4UUIIWqZsWPH0rVrV/r27Vvs+NWrV9m7dy+g22H5VnUcrBkc4MPgAB9A1yNy9Xo25+PSORefQfT1bKJTc4hJySE+PZcircL1rHyuZ1V8bM6tNGoVPs42+LrYUs/FFkd1Hp/NnIaSGkNq9CWsLI3/cRYQEMD//d//ERAQgKIoElzMSMKLEELUMsOHDy/1eEhICAA9evTAzc3tjs+jUatoVMeeRnXsGfRvoNErLNKSnJlPWk4Bqdn5uh6U7AJyCooMPSwFRVoKi7RYaNS6nph/e2RsLTWGnhoXOytc7Cyp42BdbGxNSEgI2Rf20apVK5MEF9CtpPvBBx+wcOFCWrZsyX/+8x/effddk7y2KE7CixBCCAD++ecfQHfrCKCgoIDMzMxiM5zKy0KjxtvZBm9nm0qtUe9utwWorNcOCwvj+vXrJn9toVP1VyESQghRqbRaLcePH+fPP/+kqKgI0E273rp1K6ALL8ePH6dTp048+eST5iy1TOYKL6mpqcybNw+QadLmJOFFCCFqGUVRCAwM5OGHHyYmJgaA/fv3k5GRgYeHBx07dsTS0pJz586xZs0a/vrrLzNXXJK5wsumTZtISUkBZIE6c5LwIoQQtYxGozEMyNXPONLfMgoODkatVtOmTRumT58OwJQpU8jMzDRLrWW5cOECYPrwcvPU8gYNGpj0tcUNRgsvH3zwAd27d8fOzg4XF5dyXaMoCrNmzcLHxwdbW1v69+/PxYsXjVWiEELUWvpeA/1KsUOGDOH5559n9OjRhjazZs2iYcOGREVFMXPmTLPUWZqcnBxiY2MB04eXZs2alfpnYVpGCy/5+fk88sgjPPfcc+W+5pNPPuHrr79m3rx5HDx4EHt7e4KDg8nNzTVWmUIIUSvptwnQ97z06tWLb7/9lqFDhxra2NnZMXfuXAC++uqrKnP7yNbWlqysLE6fPl2uWVGVycLCgqtXr3L58uVybQYpjMNo4eWdd95h6tSpJVZvLIuiKHz55ZfMnDmT4cOH07ZtWxYtWkRsbCxr1qwxVplCCFEr6cOLvuelLIMGDeKll14CYMKECVy5csXIlZWPjY0NrVu3Nstr+/n5yXgXM6syY14iIiKIj4+nf//+hmPOzs4EBgayf//+Mq/Ly8sjPT292EMIIcTt3XzbaMmSJezatYuCgoJS23788cd07dqV9u3bY2NT9tRnWSpfmEqVCS/x8fEAJZa89vLyMpwrzYcffoizs7Ph4efnZ9Q6hRCiJtD3vFy4cIEpU6bQu3dvDh8+XGpbKysr1q9fz5YtW/D29i52Ljo6mhdffBF7e3vGjBlj9LoB3nvvPZ5++mkOHTpkktcTVU+Fwsvrr7+OSqW67eP8+fPGqrVUM2bMIC0tzfCIiooy6esLIUR11KRJEz799FMmTZpESkoKrq6udO3atcz2derUQaPRGL7eu3cvzzzzDI0bN+brr78mOzub5cuX3/aXzcpw4cIF5s+fzy+//EJcXJxRX0tUXRVaYXf69OlMnDjxtm3u9j6gPs0nJCTg43NjmemEhATat29f5nXW1tZYW1f+BlxCCFGT2dnZ8fLLLzNr1iwAHnjgASws7vyRkJeXx4svvsgPP/xgONa7d28SExOZMmUKtra2Rqk3ISGBd999lx9++IGioiKcnZ3p1q2bUV5LVH0VCi8eHh54eHgYpZBGjRrh7e3N1q1bDWElPT2dgwcPVmjGkhBCiPLT72c0aNCgcrVXq9WGBeKCg4N544036NWrl9Hqy8nJ4bPPPuOTTz4xrDUzdOhQPvroo3vaWVtUb0bb2+jq1atcv36dq1evUlRURGhoKABNmzbFwcEBAH9/fz788ENGjhyJSqXipZde4v3336dZs2Y0atSIN998E19fX0aMGGGsMoUQotY6cuSIYZxLcHBwua6xtLRk06ZNJCQkUK9evXt6ff2WBN26dStz/6S///7b0DvUpUsXPvnkE/r06XNPrytqAMVIJkyYoAAlHtu3bze0AZRff/3V8LVWq1XefPNNxcvLS7G2tlb69eunhIWFVeh109LSFEBJS0urpO9ECCFqpsGDBxt+NleG6Oho5bvvvlN27txZrvZvvvmmAij29vbKyy+/rMTExJRoo9VqlcmTJyvLli1TtFptpdQpqqaKfH6rFKVmzW1LT0/H2dmZtLQ0nJyczF2OEEJUWWfOnGH48OG89957lTJT6JVXXuGzzz5j3Lhx/P7773ds/9577xl6VUA3q2n8+PGkpaXxww8/3NVu1qL6qsjnt4QXIYQQlWLPnj306tULZ2dnkpKSsLS0vOM1Wq2W9evX88knn7Bnzx7D8aeeeoqff/7ZmOWKKkbCi4QXIYQwuaKiInx8fEhKSmLz5s3FFh0tjz179vDFF19QVFTEV199Zdg8UtQOFfn8rjKL1AkhhKjeNBoNDz74IABr1669bduwsDCKioqKHevZsyerVq1i7dq1ElzEbUl4EUIIUWmGDx8OwJo1a8rcLiA3N5cOHTrg7e1NdHS0KcsTNYSEFyGEEJWmf//+2NnZER0dzbFjx0pts3PnTnJycrC2tqZu3bomrlDUBBJehBBCVBpbW1uCg4OxsLDg5MmTpbbZsGEDoFsYT6VSmbI8UUMYbZE6IYQQtdPnn3/O/PnzcXFxKfW8PrwMHjzYhFWJmkTCixBCiErVqFGjMs9dvHiRS5cuYWlpSb9+/UxYlahJ5LaREEIIo9Bqtaxbt67YwN1//vkHgF69eslyFuKuSXgRQghR6RRFYdSoUQwbNqzYYnM3j3cR4m5JeBFCCFHpVCoVgYGBAEyZMoUjR44A8NZbb/F///d/hinVQtwNWWFXCCGEUSiKwsiRI1m7di0NGjTg6NGjuLu7m7ssUUXJCrtCCCHMTqVSsWDBApo0aUJkZCSPPfYYWq3W3GWJGkDCixBCCKNxcXHhzz//xNbWlpCQEAYPHkx+fr65yxLVnIQXIYQQRtWuXTvmzp0LwK5duygoKDBzRaK6k3VehBBCGN3EiRNxdHTE3d0de3t7c5cjqjkJL0IIIUxi1KhR5i5B1BBy20gIIYQQ1YqEFyGEEEJUKxJehBBCCFGtSHgRQgghRLUi4UUIIYQQ1YqEFyGEEEJUKxJehBBCCFGtSHgRQgghRLUi4UUIIYQQ1YqEFyGEEEJUKxJehBBCCFGtSHgRQgghRLUi4UUIIYQQ1UqN21VaURQA0tPTzVyJEEIIIcpL/7mt/xy/nRoXXjIyMgDw8/MzcyVCCCGEqKiMjAycnZ1v20allCfiVCNarZbY2FgcHR1RqVR3/Tzp6en4+fkRFRWFk5NTJVYobiXvtWnJ+2068l6bjrzXpmOs91pRFDIyMvD19UWtvv2olhrX86JWq6lXr16lPZ+Tk5P8QzARea9NS95v05H32nTkvTYdY7zXd+px0ZMBu0IIIYSoViS8CCGEEKJakfBSBmtra9566y2sra3NXUqNJ++1acn7bTryXpuOvNemUxXe6xo3YFcIIYQQNZv0vAghhBCiWpHwIoQQQohqRcKLEEIIIaoVCS9CCCGEqFZqdXj57rvvaNiwITY2NgQGBnLo0KHbtl+5ciX+/v7Y2NgQEBDAhg0bTFRp9VeR9/qnn36iV69euLq64urqSv/+/e/4/0bcUNG/13rLli1DpVIxYsQI4xZYg1T0vU5NTeX555/Hx8cHa2trmjdvLj9HKqCi7/eXX35JixYtsLW1xc/Pj6lTp5Kbm2uiaqunXbt2MWzYMHx9fVGpVKxZs+aO1+zYsYOOHTtibW1N06ZNWbBggdHrRKmlli1bplhZWSnz589Xzpw5o0yaNElxcXFREhISSm2/d+9eRaPRKJ988oly9uxZZebMmYqlpaVy6tQpE1de/VT0vR47dqzy3XffKcePH1fOnTunTJw4UXF2dlaio6NNXHn1U9H3Wi8iIkKpW7eu0qtXL2X48OGmKbaaq+h7nZeXp3Tu3FkZPHiwsmfPHiUiIkLZsWOHEhoaauLKq6eKvt+LFy9WrK2tlcWLFysRERHKxo0bFR8fH2Xq1Kkmrrx62bBhg/LGG28oq1atUgBl9erVt20fHh6u2NnZKdOmTVPOnj2rfPPNN4pGo1FCQkKMWmetDS9du3ZVnn/+ecPXRUVFiq+vr/Lhhx+W2v4///mPMmTIkGLHAgMDlWeeecaoddYEFX2vb1VYWKg4OjoqCxcuNFaJNcbdvNeFhYVK9+7dlZ9//lmZMGGChJdyquh7/f333yuNGzdW8vPzTVVijVLR9/v5559X+vbtW+zYtGnTlB49ehi1zpqkPOHl1VdfVVq3bl3s2OjRo5Xg4GAjVqYotfK2UX5+PkePHqV///6GY2q1mv79+7N///5Sr9m/f3+x9gDBwcFlthc6d/Ne3yo7O5uCggLc3NyMVWaNcLfv9bvvvounpydPPfWUKcqsEe7mvf7rr78ICgri+eefx8vLizZt2jB79myKiopMVXa1dTfvd/fu3Tl69Kjh1lJ4eDgbNmxg8ODBJqm5tjDXZ2ON25ixPJKTkykqKsLLy6vYcS8vL86fP1/qNfHx8aW2j4+PN1qdNcHdvNe3eu211/D19S3xD0QUdzfv9Z49e/jll18IDQ01QYU1x9281+Hh4Wzbto1x48axYcMGLl26xH//+18KCgp46623TFF2tXU37/fYsWNJTk6mZ8+eKIpCYWEhzz77LP/3f/9nipJrjbI+G9PT08nJycHW1tYor1sre15E9fHRRx+xbNkyVq9ejY2NjbnLqVEyMjJ4/PHH+emnn6hTp465y6nxtFotnp6e/Pjjj3Tq1InRo0fzxhtvMG/ePHOXViPt2LGD2bNnM3fuXI4dO8aqVatYv3497733nrlLE5WgVva81KlTB41GQ0JCQrHjCQkJeHt7l3qNt7d3hdoLnbt5r/U+++wzPvroI7Zs2ULbtm2NWWaNUNH3+vLly1y5coVhw4YZjmm1WgAsLCwICwujSZMmxi26mrqbv9c+Pj5YWlqi0WgMx1q2bEl8fDz5+flYWVkZtebq7G7e7zfffJPHH3+cp59+GoCAgACysrKYPHkyb7zxBmq1/O5eGcr6bHRycjJarwvU0p4XKysrOnXqxNatWw3HtFotW7duJSgoqNRrgoKCirUH2Lx5c5nthc7dvNcAn3zyCe+99x4hISF07tzZFKVWexV9r/39/Tl16hShoaGGx4MPPsj9999PaGgofn5+piy/Wrmbv9c9evTg0qVLhoAIcOHCBXx8fCS43MHdvN/Z2dklAoo+OCqypV+lMdtno1GHA1dhy5YtU6ytrZUFCxYoZ8+eVSZPnqy4uLgo8fHxiqIoyuOPP668/vrrhvZ79+5VLCwslM8++0w5d+6c8tZbb8lU6XKq6Hv90UcfKVZWVsoff/yhxMXFGR4ZGRnm+haqjYq+17eS2UblV9H3+urVq4qjo6MyZcoUJSwsTFm3bp3i6empvP/+++b6FqqVir7fb731luLo6KgsXbpUCQ8PVzZt2qQ0adJE+c9//mOub6FayMjIUI4fP64cP35cAZQvvvhCOX78uBIZGakoiqK8/vrryuOPP25or58q/corryjnzp1TvvvuO5kqbWzffPONUr9+fcXKykrp2rWrcuDAAcO53r17KxMmTCjWfsWKFUrz5s2V/2/XjlEchIIADLuNySUkYEobO0uPYp9LmCbnyCms7byOB4gwWy0sZBuLCMN+H7xSGIaH/IhlWUbTNDFN08ET57Vn15fLJYqieDvjOB4/eEJ77/Vv4mWfvbteliW6rovT6RR1Xcfj8Yht2w6eOq89+369XnG/3+N6vcb5fI6qquJ2u8W6rscPnsg8z3++f392OwxD9H3/9kzbtlGWZdR1Hc/n8+NzfkX4fgYA5PEv/3kBAPISLwBAKuIFAEhFvAAAqYgXACAV8QIApCJeAIBUxAsAkIp4AQBSES8AQCriBQBIRbwAAKl8Ayrm5FFHwrOjAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -198,7 +206,7 @@ "for i in range(1, EPOCHS + 1):\n", " t0 = time.time()\n", " for j, (xb, yb) in enumerate(zip(x_train, y_train)):\n", - " nn, optim_state, (loss, logits) = train_step(nn, optim_state, xb, yb)\n", + " net, optim_state, (loss, logits) = train_step(net, optim_state, xb, yb)\n", " print(\n", " f\"Epoch: {i:003d}/{EPOCHS:003d}\\t\"\n", " f\"Batch: {j+1:003d}/{len(x_train):003d}\\t\"\n", @@ -209,10 +217,10 @@ "\n", "\n", "# 6) un-mask the trained network\n", - "eval_nn = sk.tree_unmask(nn)\n", + "eval_net = sk.tree_unmask(net)\n", "\n", "\n", - "y_pred = jax.vmap(eval_nn)(x_train.reshape(-1, 2, 1))\n", + "y_pred = jax.vmap(eval_net)(x_train.reshape(-1, 2, 1))\n", "plt.plot(x[1:], y[1:], \"--k\", label=\"data\")\n", "plt.plot(x[1:], y_pred, label=\"prediction\")\n", "plt.legend()" diff --git a/docs/notebooks/train_convlstm.ipynb b/docs/notebooks/train_convlstm.ipynb index f4bc827..f0a3aa5 100644 --- a/docs/notebooks/train_convlstm.ipynb +++ b/docs/notebooks/train_convlstm.ipynb @@ -25,28 +25,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "YABV1l4q2HR9", - "outputId": "2c1eb5c7-2a0e-4542-ba83-c1baf3cfdc60" + "id": "YABV1l4q2HR9" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", - " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.8/57.8 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h Building wheel for serket (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Building wheel for ml_collections (setup.py) ... \u001b[?25l\u001b[?25hdone\n" - ] - } - ], + "outputs": [], "source": [ "!pip install git+https://github.com/ASEM000/serket --quiet\n", "!pip install optax --quiet\n", @@ -162,7 +143,7 @@ "height": 1000 }, "id": "c_mHfJYd2A68", - "outputId": "bb25c7df-fd0f-410a-f679-a468dba2e5ea" + "outputId": "6cc2ac5e-7add-4429-9d83-89a0894c62bd" }, "outputs": [ { @@ -205,54 +186,27 @@ "outputs": [], "source": [ "class Net(sk.TreeClass):\n", - " def __init__(self, hidden_features: int, *, key: jax.Array):\n", - " k1, k2, k3, k4 = jr.split(key, 4)\n", - " self.convlstm1 = sk.nn.ScanRNN(\n", - " cell=sk.nn.ConvLSTM2DCell(\n", - " in_features=1,\n", - " hidden_features=hidden_features,\n", - " kernel_size=5,\n", - " padding=\"same\",\n", - " key=k1,\n", - " ),\n", - " return_sequences=True,\n", - " )\n", - " self.convlstm2 = sk.nn.ScanRNN(\n", - " cell=sk.nn.ConvLSTM2DCell(\n", - " in_features=hidden_features,\n", - " hidden_features=hidden_features,\n", - " kernel_size=3,\n", - " padding=\"same\",\n", - " key=k2,\n", - " ),\n", - " return_sequences=True,\n", - " )\n", - " self.convlstm3 = sk.nn.ScanRNN(\n", - " cell=sk.nn.ConvLSTM2DCell(\n", - " in_features=hidden_features,\n", - " hidden_features=hidden_features,\n", - " kernel_size=1,\n", - " padding=\"same\",\n", - " key=k3,\n", - " ),\n", - " return_sequences=True,\n", - " )\n", - " self.conv = sk.nn.Conv2D(\n", - " in_features=hidden_features,\n", - " out_features=1,\n", - " kernel_size=3,\n", - " padding=\"same\",\n", - " key=k4,\n", - " )\n", + " def __init__(self, features: int, *, key: jax.Array):\n", + " k1, k2, k3 = jr.split(key, 3)\n", + " self.convlstm1 = sk.nn.ConvLSTM2DCell(1, features, 5, key=k1)\n", + " self.convlstm2 = sk.nn.ConvLSTM2DCell(features, features, 3, key=k2)\n", + " self.conv = sk.nn.Conv2D(features, 1, 3, key=k3)\n", "\n", " def __call__(\n", - " self, input: Annotated[jax.Array, \"f32[F,1,H,W]\"]\n", - " ) -> Annotated[jax.Array, \"f32[F,1,H,W]\"]:\n", - " input = jax.nn.relu(self.convlstm1(input))\n", - " input = jax.nn.relu(self.convlstm2(input))\n", - " input = jax.nn.relu(self.convlstm3(input))\n", - " input = jax.vmap(self.conv)(input) # vectorize over frames\n", - " return jax.nn.sigmoid(input)" + " self, input: Annotated[jax.Array, \"Float[F,1,H,W]\"]\n", + " ) -> Annotated[jax.Array, \"Float[F,1,H,W]\"]:\n", + " # F: number of frames\n", + " # C: number of channels\n", + " # H: height of the frame\n", + " # W: width of the frame\n", + " # initialize state for the cells by passing sample input\n", + " state = sk.tree_state(self, input=input[0])\n", + " output, _ = sk.nn.scan_cell(self.convlstm1)(input, state.convlstm1)\n", + " output, _ = sk.nn.scan_cell(self.convlstm2)(output, state.convlstm2)\n", + " # vectorize convolution over frames\n", + " output = jax.vmap(self.conv)(output)\n", + " # apply sigmoid to get values between 0 and 1\n", + " return jax.nn.sigmoid(output)" ] }, { @@ -272,38 +226,36 @@ "base_uri": "https://localhost:8080/" }, "id": "9igKcrLM2A69", - "outputId": "cfb76f8b-a239-49db-83fe-10994c9fe9b6" + "outputId": "182e3f31-0955-47a8-de86-3c7b7ee94877" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "┌──────────┬───────────────────────┬───────┬────────┐\n", - "│Name │Type │Count │Size │\n", - "├──────────┼───────────────────────┼───────┼────────┤\n", - "│.convlstm1│ScanRNN[ConvLSTM2DCell]│416,256│1.59MB │\n", - "├──────────┼───────────────────────┼───────┼────────┤\n", - "│.convlstm2│ScanRNN[ConvLSTM2DCell]│295,168│1.13MB │\n", - "├──────────┼───────────────────────┼───────┼────────┤\n", - "│.convlstm3│ScanRNN[ConvLSTM2DCell]│33,024 │129.00KB│\n", - "├──────────┼───────────────────────┼───────┼────────┤\n", - "│.conv │Conv2D │577 │2.25KB │\n", - "├──────────┼───────────────────────┼───────┼────────┤\n", - "│Σ │Net │745,025│2.84MB │\n", - "└──────────┴───────────────────────┴───────┴────────┘\n" + "┌──────────┬──────────────┬───────┬────────┐\n", + "│Name │Type │Count │Size │\n", + "├──────────┼──────────────┼───────┼────────┤\n", + "│.convlstm1│ConvLSTM2DCell│105,728│413.00KB│\n", + "├──────────┼──────────────┼───────┼────────┤\n", + "│.convlstm2│ConvLSTM2DCell│73,856 │288.50KB│\n", + "├──────────┼──────────────┼───────┼────────┤\n", + "│.conv │Conv2D │289 │1.13KB │\n", + "├──────────┼──────────────┼───────┼────────┤\n", + "│Σ │Net │179,873│702.63KB│\n", + "└──────────┴──────────────┴───────┴────────┘\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Loss: 6.7986e-01: 100%|██████████| 1000/1000 [01:20<00:00, 12.39it/s]\n" + "Loss: 6.7987e-01: 100%|██████████| 1000/1000 [00:33<00:00, 30.02it/s]\n" ] } ], "source": [ - "config.hidden_features = 64\n", + "config.features = 32\n", "config.epochs = 1000\n", "config.key = jr.PRNGKey(0)\n", "config.optim = ConfigDict()\n", @@ -317,21 +269,21 @@ "config.optim.scales = [0.5, 0.5, 0.5]\n", "\n", "\n", - "def train(config: ConfigDict)->Net:\n", + "def train(config: ConfigDict) -> Net:\n", " lr = optax.piecewise_constant_schedule(\n", " init_value=config.optim.init_value,\n", " boundaries_and_scales=dict(zip(config.optim.boundaries, config.optim.scales)),\n", " )\n", " optim = getattr(optax, config.optim.kind)(learning_rate=lr)\n", - " net = sk.tree_mask(Net(hidden_features=config.hidden_features, key=config.key))\n", + " net = sk.tree_mask(Net(features=config.features, key=config.key))\n", " optim_state = optim.init(net)\n", "\n", " print(sk.tree_summary(net, depth=1))\n", "\n", " def loss_func(\n", " net: Net,\n", - " xb: Annotated[jax.Array, \"f32[N,F,1,H,W]\"],\n", - " yb: Annotated[jax.Array, \"f32[N,F,1,H,W]\"],\n", + " xb: Annotated[jax.Array, \"Float[N,F,1,H,W]\"],\n", + " yb: Annotated[jax.Array, \"Float[N,F,1,H,W]\"],\n", " ):\n", " net = sk.tree_unmask(net)\n", " logits = jax.vmap(net)(xb) # vectorize over the batch dimension\n", @@ -341,8 +293,8 @@ " def train_step(\n", " net: Net,\n", " optim_state: Any,\n", - " xb: Annotated[jax.Array, \"f32[N,F,1,H,W]\"],\n", - " yb: Annotated[jax.Array, \"f32[N,F,1,H,W]\"],\n", + " xb: Annotated[jax.Array, \"Float[N,F,1,H,W]\"],\n", + " yb: Annotated[jax.Array, \"Float[N,F,1,H,W]\"],\n", " ):\n", " loss, grads = jax.value_and_grad(loss_func)(net, xb, yb)\n", " updates, optim_state = optim.update(grads, optim_state)\n", @@ -375,10 +327,10 @@ "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 440 + "height": 439 }, "id": "ALAbakDJ2A69", - "outputId": "07b80e11-8e15-4401-e211-006f4dc84bf9" + "outputId": "5697a561-9a81-43d6-8790-8a712b46d34a" }, "outputs": [ { @@ -390,7 +342,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWIAAAGVCAYAAADEy/vbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqVklEQVR4nO3da3Bb5Z3H8d/RzbYsy5IsyZZvcRw7FxMg1y6UkBBIAu0uGXoBBhYyMG13gd2Wmd1uX3R2tu/YZbpDKR2G7LalbEImDZSGbqaBNOQCKZB7bCfxLb47vkiyZNm6WrL07AtqLSYOkRMpj2z/PjNnmlqy/c/B+Vo659GRIoQQICIiaVSyByAimu8YYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJNKneUVGUTM5BRDQnpfLiZT4iJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpJMI3sAIspOGo0GOTk5UKvVMBqNMBgMiEajcDqdCAaDssebUxhiIrqCoigwGo1YuHAhjEYj1q1bhzvvvBO9vb3Yvn076uvrZY84p/DQBBFNoSgKACAnJwdWqxUlJSW4/fbbsXnzZnz1q1+F1WpN3ofSg4+IaU6qrq7GypUroVKpcO7cObS3t8seaVbQ6/VYtWoVFi5ciMLCQlRUVMBgMKCyshKKosBkMuG+++5DSUnJtJ/vdDpx+vRpjIyM3OTJZzeGmOYcRVGwatUq/PjHP4ZWq8WLL76Ijo4OCCFkj5b1CgsL8fjjj+PBBx+ERqOBVquFSqVCXl4e1Go1HA4Hvve97yEcDk/7+SdOnEB/fz9DPEMMcZbQaDTQ6/VQq9UYHx9HOBxmOFKUl5eH3Nzc5NNlRVFgs9lQWloKrVYLm82GoqIiJBIJAIAQApFI5Koxmc80Gg0sFgvKy8uventRUdFVP7+7uxtWqxVms5n7eAYY4ixRWVmJb3/72ygrK8PRo0fx3nvvIRKJyB4r62m1Wtx7773YtGkTNJrPfpxVKhWWLFmCgoICqFQqbN26FTU1NckQJxIJHDlyBO+99x7Gx8dljj/nVFdX47nnnsPQ0BAOHz6MAwcOcB+ngCHOEg6HA9/+9rdx2223IRaL4fDhwwxxCjQaDdauXYu/+7u/Q25ubvLjiqIkHyFv2LAB69evT94Wi8UQiUTwwQcfMBJpVlpaiocffhjRaBShUAiHDx/mPk7BdYdYp9OhsrISVqsVIyMj6OnpYThSZLVasWDBAmi12uTHli9fjoKCAmg0GpSVlWHt2rUIBALJ20dGRtDd3c0f6mkoigKVSgWVavpFQJ+PMvBZvEtLS7F27VqMjIygt7cXXq/3Zo07503+t+DKitSlHOLJnTp53NJiseDpp5/Gpk2bcOzYMbz88su4fPlyZqacY9asWYPvf//7sFqtyY8VFBSgsrISarUa99xzD2praxGPx5O3f/jhh/j5z3+O/v5+GSPPKSqVCps2bcKyZcvQ29uLV155BUePHpU91pzCCM9MyiFWq9XJPwshoNfrUVtbi6985StwuVzIz8+HWq1GIpHgSaZrsFqtWLFiBUpLS6e9vaSk5IrlQQMDA1OeetP1UxQluY/NZjPMZrPskWieSznE//AP/5D8sxACFosFtbW1AICqqio88cQTGBgYwKlTp3D27NnkiREiIvpyKYf4Jz/5CYD/f8oxubYQAJYsWYIf/OAHGBsbw0svvYTGxkZEo9EMjEtENPekHOIve/qm1Wqh1Wqh0WhQUlKC8vJyBINB+Hw+nlxKk7y8PDgcDkQiEYyNjcHv98seaU7QaDSwWq2oqKhAKBSCz+ebcmyerp/RaERZWRnGxsYwMjLCk/lfIq3L13Q6He6//34sWLAAXV1d2LlzJ5qamtL5LeatW265BT/60Y8wPDyMvXv3Yv/+/QxGGlgsFmzbtg0bN27EqVOn8Oabb8Ltdssea9abPOlss9nQ39+PN998E2fOnJE9VtZKa4g1Gg1uv/123HbbbWhoaMCBAwcY4jQpLy9HeXk5AoEAWlpa8P777zPEaWAwGLBu3ToAn13kZu/evQxxGqhUKtTV1aGurg4dHR04cuQIQ/wl0hrieDyOjo4OdHZ2or29nWsz08jpdKKlpQUejwednZ08GZom4XAYLS0tGBoawrlz5/iS3DQRQqCrqwsdHR3o6+uDy+WSPVJWS2uIo9Eo/vjHP+L1119HIBDgzk+jCxcu4MUXX0RPTw+Gh4f5aDhNPB4PXn/9dRw8eBCBQIAPHtJkYmIChw4dwvbt2+Hz+diCa0g5xF98pKAoCrRabXLtcCwWQzAYxODgINra2rhqIs38fj86OzvR0dEhe5Q5JRqNor+/H62trbJHmXM8Hg/a2tqmvEKUppdyiH/6058C+OwpRyKRgMlkwubNm7F8+XJcunQJ77//PoaGhvDpp5/y0RoR0QykHOL//M//BPDZlavi8TgqKytRWVmZDPEvf/lLtLe3Y2JigiEmIpqBGR+aSCQSSCQSCAQC6OnpQVNTE7q7u+H3+7lmOEWjo6O4dOkSRkdHkx/Ly8tDSUkJcnNz4fV64Xa7p/xC6+vr4+GeNPJ4PBgeHkZPTw/XZGcAL3MwMymHeDIKkzvY6/Xif/7nf3DgwAG4XC4MDw9nZsI56PTp0/jJT36CnJyc5CsV6+rq8Oyzz6K6uhpHjx7Fzp07EQqFkp/DfZw+ExMTOHz4MHbt2pU8jknpI4RgiGco5RB/ccdGIhE0NjaisbEx7UPNdYODgxgcHJzysWAwiL/9279FIpFAd3c3Dh8+jLGxMUkTzm2T+/jQoUM8kZQBkyFmjFPHC8NnCZfLhf379+PChQs4ffo0YrGY7JFmhXg8jvr6euzevXvK9Z2rqqqwYsUKqFQqNDY2Tnnz0Hg8jrNnz2JiYkLGyHOa0+nEqVOn4Ha7UV9fz5/jFDHEWaK7uxu/+MUvoNVqEQwG+br8FMViMXzwwQc4ceLElGvgPvTQQ6ioqIBWq8Xvfvc77NmzJ/kITQiBQCDAcxoZ0NXVhVdeeQUXL15EIBDgeY0UMcRZIhqN8qW110EIAb/ff8UJt6GhIXg8Hmg0GgwNDaG/v59PlVMwuTz182+0KoRIvutGPB5HOBxOPtL94j51u90YHBzEwMDATZ99NmOIaU5qaGjAz372M6hUKpw5c4YRTtHkiiiv14uJiYnkMzOTyYTCwkK4XC787//+L1pbWyGESB7emdy//f39GBoakjb/bMUQ05zU3t6Ozs5OAFxKNRNCCITDYYyOjiIajcLv90MIAY1GA6PRCI/Hg/379+PDDz9ELBZDNBqdcsgHAK+Dch0YYpqzGITr8/mwTrf6YfJFXZP/y190N04R3ItERFJN//7jRER00zDERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERLKJFAHgxo0btzmxPfTQQ6Kjo0OMjY2JH/7wh0Kr1Wbse6WC79BBNEuoVCooipJ8g0/6cpP7azqKoiRvU6lUUKvVV92niUQi4+9CwhATzQJWqxV33303ysvLceHCBRw/fhzhcFj2WFmrtLQU69evh91un/b22267DUajETqdDnfddRdisRji8fgV9wuFQvjzn/+MlpaWjM7LEBPNAiUlJXj66adx5513YteuXWhsbGSIv0RlZSWeeeYZ3H777dPertFooNfroVKpsHnzZqxfv37a+zmdTgSDQYaYZieVSoXCwkLk5+djfHwcPp8PsVhM9lizQm5uLsxmMzSa///nWVZWBqvVCrPZDLvdjoqKCuj1+uTtkUhk3u/jz//MlZWVoaioCCaT6Zqfl5eXh7y8vGlvi8ViKCkpQUVFBSKRCEZGRjAxMZHmyWfw5qFXO9ZCNB2j0YhHH30U99xzD5qbm7Fjxw709vbKHmtWWL16NZ544gkUFxcnP2Y2m7Fq1SrYbDZ0dnaisbERkUgkefvFixexY8cO9PX1yRg5KxgMBjz88MPYtGkT7HY7Vq9eDbPZfENfMxKJoL6+Hl1dXTh//jx27tyJy5cvz+hrpJRYrprglonNbreLX/3qVyKRSIgjR46I22+/XfpMs2XbunWraG9vT/WfphBCiEOHDonly5dLn13mVlRUJF577TURj8dntO9S9ac//UnccsstM54rFTw08ReKomDBggWora1FNBpFS0sLnE6n7LFmhYKCAixbtgxFRUXJj5lMJpSXl0NRFFgsFqxbtw6lpaXJ2yePu7lcLhkjZ53c3FwsXboUDocDq1evnnLYIRVFRUW4++67UVZWho6ODnR2dnJlxSzCEP+FWq3Gpk2b8Oyzz8Lr9eLFF19kiFPkcDjw3HPP4Y477kh+TK1Ww2azAQBqamrwwx/+EOPj48nbe3p68B//8R8M8V9YLBY89dRTeOCBB5Cfnw+r1Tqjz6+trcWPfvQj+P1+vP7669i+ffuUQxeU3eZ9iFUqFbRaLXJyclBcXIzFixfD7XbDYrEgLy8P8XgcsVgs4+sIZzOdToeKigosWbJk2tv1ej2qqqqmfEytVsNgMNyE6WYHrVaLsrKyq+7Da5ncx9FoFEVFRVCp+KLZ2WTeh7iiogJf+9rXUFZWhjvvvBM6nQ5msxnf+ta3cMstt6CpqQkHDhyAz+eTPSoRzVEMcUUFnnrqKaxYsQJqtRoajQZarRbf+MY3EI/H8e677+L48eMMMRFlzLwMsVqtht1uR2FhIaqrq2E0GpGTk5O8XVEUaLVaaLVaFBUVoba2Frm5uRgeHobH45E4efZQFAU2mw0WiwU1NTXIz8+f0efrdDqUl5ejrq4OY2NjcDqd83INrMVigc1mQ0VFBYxG4w1/vcn/LkuXLsXo6CicTicCgUAaJs1ehYWFsNlsyXXWmVpqazAYsGjRIsTjcXg8HgwPD6ftkOW8XEdsNBrx9NNP44EHHkBRURGWLl2KgoKCae/rdDrR2toKj8eDXbt24d133532pZDzTU5ODh5//HF885vfhMViwdKlS2GxWFL+/FAohNbWVrjdbnz00Uf45S9/Oe9O3KnVamzduhXbtm1DUVERFi9ePGXt8PUQQqCrqwtdXV3o7e3Ff/3Xf+HEiRNpmjg7bdq0Cd/5zndQXFyM2tpalJeXZ+T7eDwetLa2wuv14u2338aePXumnIC+mlQSOy8fEet0OtTV1WHLli3XPKlRXFwMu92OsbExHDt2bE79QroRarUaNTU12Lx585RnE6nS6/VYuXIlAGB0dBS5ubnpHjHrKYqCqqoq3HvvvWl5NDz5Naurq1FdXY22tja88847afm62czhcGDDhg1wOBwZ/T5FRUX46le/ikgkgnPnzqX1hOi8DPH4+DiOHz+OvLw8FBcXY+3atVd9BU5vby/OnDmD4eFhtLS0cG3mX8TjcdTX12P37t2wWq1Yu3btjB7N+f1+nDlzBpcvX8bJkycRDAYzOG12SiQSaG5uxp49e2C1WrF69WpUVlbe8NdsamrChQsXcPnyZfT396dp2uzV2dmJd955B8XFxVixYgVqa2sz8n2GhoZw+vRpuN1uNDQ0pPeZcaqvKkEWvHImXZtKpRImk0mUlZWJb3zjG+L8+fNX/Xv/8Y9/FH/1V38lHA6HMBgM0mfPlk1RFGE0GkVpaam47777xLFjx1L9URJCCNHV1SWeeuopUVZWJiwWi1Cr1dL/TjI2g8EgHA6H+MpXviL27ds3o304nfHxcfHzn/9c1NTUiOLiYpGbmyv975jpLS8vT5SUlIjly5eLHTt2iEQiccP7cTofffSR2LhxoygtLRUFBQVCUZSU5kvFvHxEnEgk4PP54PP54HQ64fF4MDIygtzcXOTl5SGRSCAUCiEajcLtdmNwcBCDg4Oyx84qQgiMjY1hbGwMFosF0Wh0Rp8/MTEBj8czLx6xfZlAIIBAIACdTpe2F2CMjY1hYGAAoVAoLV8v24XDYYTDYcRiMQSDQQghMnIIMRKJwOVyYWBgIO1fe16G+PO6u7vx2muvobi4GPfeey8eeOAB+P1+7N27Fw0NDejo6MDIyIjsMYloDpv3IR4YGMDbb78NnU4HvV6P++67D2NjY3jvvffwhz/8AUIIvqqOiDJq3ocY+OxQxcTERPLE0cjICDweD0/MpSgcDqOpqWnKhWq0Wi0qKiqSK066u7uTT5WFELh8+TKfaXzO+Pg42tracPz4cZjNZlRVVc1oNcrkPvb5fLh8+TJ/dmcZhvgv4vE4Dh06hObmZsRisXl9XdeZGhgYwKuvvjplCZbZbMZzzz2HBx98EJcuXcLPfvYzXLp0KXl7JBLh9Yk/x+v14je/+Q3+8Ic/YP369Xj++edntB62ra0NL730Ei5duoShoaEZH7MnuRjivxBC8KTcdQqHw1e8lYzNZoPT6UQ8HofP58P58+fR2NgoacLsF41G0d7eDuCzteuhUGjK8ihFUZLrVsU0bx46uY8vXLhw84bOMpP7ZXK/pWud7+Sbh2byTUQZYsqIcDiMw4cPIxQKob29HcPDw7JHmjW6urrw5ptvTrkUps1mS7556MWLF/Hpp59Oec+6tra2ef/y+0gkgqNHjyIWi6GsrAwbNmxIXor1eoVCIXz88cdobm5GS0sLvF5vmqb9glTX0CEL1gtymz2boihCr9cLk8kkDAbDvF0nfD2bVqsVRqNRmEym5LZu3Tpx7NgxkUgkxI4dO0RNTc2U2w0Gg1CpVNJnl7kpiiLy8vKEyWQSX/va10R9ff0MVwpfyel0imeffVaYzebr3sep4CNiygghBEKh0LxZy5pOsVjsigsgeb1eDA4Ooq+vD0NDQxgZGeEVAb9ACJFcU+x2uzEwMHDV65/o9XqYTCaoVCqMjY3B7/dPe9jB7XbD7XZn/MTyvLzoD9FsYzKZsHr16uSbhzY0NKR0wZn5ymazYdWqVVcN8Zo1a/DEE0/AaDTi97//Pd57771pX7IcDodRX1+P7u7u654lpcSm+hAdWfDUgxs3btzSsT300EOio6ND+P1+8S//8i9Cq9Vm7Hvx0AQR0TSGhobw4YcforCwMCveaJWHJoho3jEYDLDb7VCpVPB6vZlbDQGkdGiCISYiyqBUEsu3eiUikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSTJPqHYUQmZyDiGje4iNiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLZRIoAcOPGjVvWbU8++aQYGBgQw8PD4plnnhEqlUr6TJ/fUpHyO3QQUfqoVCooioJEIsF3v0nB5P76IkVRoFarr/j/090XQNbub4aY6CYrKSnBhg0bYLVa0dDQgJMnTyIajcoeK2stWLAA69evh9lsvuI2RVGwZs0a5OfnQ6PRYOPGjcjJyUEikbjivmNjYzh27Bg6OjpuxtgzwhAT3WTl5eX43ve+h1tvvRX//d//jYaGBob4S9TU1OAf//EfsXjx4mlv1+l0yMvLAwD8zd/8DTZv3jzt/Xp6ejA8PMwQ0+ykVqthMpmg1+sRDofh8/kwMTEhe6xZQa/Xo7CwEBrN//9TKy8vR1FREcxmM+x2O8rKyhAMBpO3h8NhjIyMIB6Pyxg5K6jVahQWFiI/Px8OhwMWiwUmk+man5eXl5eM8hf5/X44HA5UVFQgFArB5/NlzT5miOmazGYzHn/8cdxxxx04d+4cdu7ciaGhIdljzQorV67E448/PuVptdVqRUVFBdRqNdavXw+LxYJYLJa8/cyZM9i5cydcLpeMkbOC0WjEY489hrvuugsOhwN2u/2Gv6bZbMa2bduwceNGnDp1Crt27cqefcxVE9yutVVWVoo9e/YIIYTYt2+fWLx4sfSZZsv22GOPib6+vlT/mQkhhHj33XfFokWLpM8ucysrKxM7d+6c0X6bib1794rq6uqb8ndJxZx8RKxSqVBdXY1FixYhGAyiubkZHo9H9lizgslkwtKlS6c8grPb7XA4HMk/r1+/HosWLUre7vf70dTUBK/Xe9PnzUb5+flYtmwZbDYbVq5cedWnyldTXFyM9evXo7q6Gh0dHejq6srKM/2ZplLNn5c5zMkQ63Q6fP3rX8d3vvMd9PT04IUXXmCIU1RVVYV/+qd/wm233Zb8mFarhc1mAwDU1dXhxz/+8ZSTS62trXjhhRdw4sSJmz5vNiouLsbf//3f4+6774bRaERhYeGMPn/58uX413/9V4yOjuK1117DG2+8MeXQBc09cyrEKpUKWq0Wer0eDocDS5YsgUajgdlsRl5eHiYmJvgDfQ25ublYsGABlixZMu3tBoMBBoNhysei0Sj0ev3NGG9W0Ol0qKiouOo+vJbJfRwIBFBUVJTm6SgbzakQV1dX4+tf/zpKSkpw1113QaPRwGazJU801dfX4+DBgwgEArJHJUrZ1V6cQHPHnApxTU0Nvvvd72Lx4sXQaDRQq9WwWq149NFHEY/H8eabb+KTTz5hiIkoq8z6EGs0GhQXF6OwsBBVVVUoKChATk5O8nZFUaDVapPHOZcsWYLCwkK4XC74fD55g2cRlUoFu90Ok8mEhQsXzvjkUk5ODqqqqrB06VKMjo7C5XJlzfrMm0VRFBQVFcFqtaKmpgb5+fk3/DXVajXsdjuWLVsGv9+PoaEhhEKhNEybvSbXVjscDhiNxox9n4KCAtTU1ECr1WJ4eFj6OSRFpHg6NlufHlmtVnz3u9/FPffcA5vNhqVLl171eOXAwADa2trgcrnwxhtv4P3335+XZ6O/yGAw4Mknn8TWrVthsViwdOnSGf0jCAQCaGlpgcfjwYEDB/Cb3/xm3v2S02q1eOSRR/Doo4/CbDZj2bJlN3x8Nx6Po7OzE93d3Whvb8f27dvR2NiYpomzj6Io+Ou//ms89dRTsFqtWLx4cXK1Trq5XC60trbC4/Fg165d2Lt3b8YePKTSmFn/iDgnJwe33nortmzZcs1fFqWlpXA4HHC73Th48OBNmjD7aTQaLFu2DJs3b55yAZVUGQwGrFmzBkII9Pf3Q6fTZWDK7KZSqbBo0SJs2rRpxs8orkatVqO2tha1tbWw2WzYs2dPWr5utlIUBRUVFdi4cSMsFktGv5fdbofdboff78fHH38MlUol9VncrA9xKBTCn//8ZyQSCZSVlWHNmjUoKCiY9r6dnZ04e/YsXC4X2tvbb/Kk2SsajSZfaVRcXIw1a9bM6NGcz+fD6dOnMTg4iE8//RSRSCSD02anRCKBxsZG7N69GzabDWvWrLnhR3MTExM4f/48mpub0dnZCafTmaZps5MQAm1tbXjrrbdgt9uxatUqVFVVZeR79fX14cyZMxgeHsbFixenvUjQTZXqK1GQBa+2mW5TqVTCbDaLsrIysW3bNtHR0XHVv8Nbb70lVqxYIRwOh9Dr9dJnz5ZNURRRWFgoSktLxYMPPijOnTuX6o+FEEKI5uZm8cgjj4iysjJhMpmy7nqwN2srKCgQpaWlYsOGDeLIkSMz2ofTCQaD4oUXXhBVVVXCbrcLnU4n/e+Y6S0/P184HA6xevVq8fvf//6G9+HVvP/+++KOO+4QDodDGAyGjP6dUjHrHxEnEgmMjIxgZGQEg4ODGB4eTq4bzs3NRTweRygUQiwWg9vtxsDAQPa8vjxLCCEwOjqK0dFRVFZWzvhKYJP7tr+/P0MTzg5+vx9+vx9GoxHj4+M3/PUSiQTGxsYwMDAwb67OFgwGEQwGoVKpEA6HM/Z9wuEwXC4XBgcHM/Y9ZmLWh/jz2tra8Morr8But2PLli3YtGkT3G43fve736G1tRUtLS1cukZEWWdOhbinpwd9fX3Q6/WwWCzYuHEjPB4P9u3bh0OHDkEIwVUSNOvwZ3bum1MhBj57OheLxdDb24sTJ06gt7cXIyMj8g/GzxKBQADnz5+fsr90Oh0WLFiAoqIi+Hw+dHd3J0/ICSHQ0dGBsbExWSNnnXA4jObmZhQWFsJqtaKysnJGK0lGRkbQ3d0Nn8+H/v5+hngemHMhBj47Zrl//36cO3cOkUgEvb29skeaNbq7u/HSSy9NuZ5ESUkJfvCDH+C+++7DhQsX8PLLL6Ovry95eygU4j7+HKfTie3bt2P37t24//778f3vfz950aRUnD9/Hi+//DJ6e3sxMDAw714cMx/NyRAnEgn09/fP+5NH1yMQCKCpqWnKxyorK+F2uxGPx+H1etHQ0MDlf18iEomgtbUVALBo0SKEw+EpMVUUJXmJRyHEFc/WPB4PGhoa0NnZefOGzjJCCMTjccTj8Sn760ZNvnloIpHIqmfJczLElF5+vx8HDhyAy+XCxYsXMTo6KnukWaOtrQ1vvPHGlLf5cTgc2LBhA2w2G+rr63HixIkpqyLOnz8/7w/1BINBHDx4EF6vF1VVVVd989CZGBsbw0cffYTOzk40NjbC7/enado0SHXdHbJgjSE3OZuiKCI/P1+YTCaRn58/b9cJX8+m1WqF0WgUJpMpud1///3i7NmzIhaLiVdffVVUVlZOuZ37+LOfOb1eL0wmk/jWt74lWltbZ7pU+Ard3d3iySefvOn7OBV8REzXJISY8uaWlLpYLHbFNbCHh4fR398Ps9mMoaEheL1eLqv8AiEEQqEQQqFQcn9N99JxRVGg1+uTzzhGR0cRDAanPcHZ398Pt9udlddBmfUX/SGabaxWK1atWgWz2YzW1lZcvHiRb1jwJRwOB1atWnXVC1HdfffdeOyxx6DRaPDWW2/hgw8+mPZ+gUAA586dw+XLlzM57hVSSmyqD+uRBU9XuHHjxu2L27Zt28TAwIAYHh4WzzzzTNYd1uGhCSKa8/r7+3Ho0CHodDr09PTMynXXPDRBRLOa0WiEzWaDoigYHh7OumPAqSSWISYiyqBUEpueVdJERHTdGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIsk0qd5RCJHJOYiI5i0+IiYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpIs5esRK4qSyTmIiOakVK7lzkfERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkjHERESSMcRERJIxxEREkmlkD0BE2UWlUkFRlGveTwiBRCJxEyaa+xhiIkoyGAxYt24dFi9efM37NjU14ZNPPkEoFLoJk81tDDERJRUWFuLhhx/GN7/5zS+9nxACu3btQkNDA0OcBgyxBIqiQKX67PB8IpGAEELyRLOXoigoKCiA0Wic9ul0PB7H6OgogsGghOlmH5VKhfz8fJhMpmvet7i4GGVlZdBqtdzHN4ghlkCn08FgMEBRFASDQYTDYdkjzVparRZbtmzBgw8+CK1We8Xtfr8fv/3tb3H06FH+wkuzNWvW4N/+7d/gcrmwZ88e7uMbwBBLoNVqkZ+fDwAYHx+XPM3splarceutt+KRRx5Bbm7uFbcPDw/jzJkzOHr06M0fbo5buHAhFi5cCLfbzX18g1IOcUlJCQAgGAwiEAjwN98NKC4uxpo1a5Cbm4tAIJA8xiaEgBACPT09aG9vx8TEhORJZz+dTofly5fjgQcegNfrRXNzM8bGxmSPRTRFyiFeuXIlhBDo7u5mJG7Q8uXL8fzzz6O4uBjxeDx5nFgIgYmJCezatQuvvvoqAoGA7FFnvfz8fDzyyCPYsmULTp06hX//939HU1OT7LGIpkg5xEVFRRBCwOfzQa/XIxqNIhaLIR6PZ3K+OamgoAALFixAaWnplI8LIRCLxWC325Mn8+jGqNVqFBcXo7i4GE6nc9rDF0SypRzihx56CEIIeL1euFwueL1e/OlPf8LFixczOd+8oihKSgvpiWhuSTnEW7duBTD1OGZvby9DnGaMMdH8k3KIv7g0yGAwoLKyEkuXLkUgEIDT6UQsFkv7gPONoiiwWCxYvHgxRkZG4HK54Pf7ZY81J+Tn56O6uhqhUAherxfDw8N8iW4aqNVqOBwO1NXVIRAIYGhoiKuBZui6l6+ZzWY88cQTuPfee3Hy5En86le/wuDgYDpnm5dUKhXWr1+PkpIS9Pf349e//jU++eQT2WPNCdXV1fjnf/5neL1e7N27F7t27eIa7jQoKCjAo48+irvuuguNjY147bXX0NnZKXusWeW6Q5ybm4tbb70VdXV1GB8fh16vT+dc85aiKKisrERpaSm6u7uxb98+2SPNGWazGXfccQdisRguXrwItVote6Q5QavVoq6uDnV1ddDpdCgoKJA90qxz3SEOBoM4ffo0urq6UF9fz6fPaSKEQHNzM+rr6zE4OIi+vj7ZI80ZLpcLp0+fhtvtxtmzZ7kEM03Gx8dRX1+PS5cuoampCSMjI7JHmnWuO8Q+nw+//e1vsW/fPkQiEYyOjqZzrnkrHo/jk08+wUsvvQSfz8f9mkZdXV34xS9+gcbGRgQCAR7HTJNAIIB33nkHu3fvZguu04xDHIlEEA6H4fF44HQ6MTAwkIm55rVgMIjBwUG+AizNxsfH4XK5+DObZolEAj6fD/39/XzF7XVKOcSTF4H+9NNPsX//frhcLly4cCGTs81Lk8sDiWj+mFGI4/E4Ghoa8Otf/xqjo6Nc+pNmDDDR/JRyiI8fP454PI6uri5Eo1FGOA0mJibQ19cHp9OZjPDExAS6u7v50vE0mZiYQG9vL1wuFy5evMhr5lJWSjnEzz//PIDPzjxHIpGMDTSfBINBvP3229i7d2/yF1sikYDT6eQ+TpNgMIi33noL7777LkZHR7kKhbJSyiE+ffp0JueYVyYP84yPj6OrqwsnT57kM4w0+eI7nkQiEXR2duLkyZM89JNmk+eN4vE49+0N4oXhJWhtbcUbb7wBjUaDCxcu8Ic4TbxeLz766CP09vYmPxYIBHg9lAxpaWnBxx9/jKGhIf4c3yCGWIKGhga0tbVBURSEw2H+AKeJy+XCjh07cOTIkeTHhBDcxxly7tw5/PSnP4XT6eRLxW8QQyxBLBbjBZLSZPIa2ZcvX0Z/fz/cbjd8Pp/sseaESCQCn8931Z/VoaEheL1evoAjDRhimtVisRgOHjyI/v5++P1+tLe3yx5pzmhpacGOHTswNDR0xW1CCHR1dfFdZNKEIaZZLR6P48KFC3xxUQYMDAxg//79aG1tlT3KnMcQE1FSOBxGfX09jEYjTp8+zUe8N4kiUjyLwXeNIJr7NBoN7HY7CgoKEAwG4XK5EI1GZY81q6WSWIaYiCiDUkks3yqYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISLKUr77GdzggIsoMPiImIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKS7P8APLsf+5kI3jYAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWIAAAGVCAYAAADEy/vbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAr7klEQVR4nO3daXBb12H28edeLAR3bNwXUQxFUYqsWJsjL2Eo2bGU6chNo1q108p2Mk7a2O10kg+eaadNv3TSsad139jT2ImmTixLduU4jjxqKkuulspKJFsLKZIiKVJcxQ0EQZAECIBY7nk/2EBEWwsoAToA+fxm7lgGuBweQX9eXBzcqwghBIiISBpV9gCIiBY7hpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMn28H6goSjLHQUS0IMXz5mXuERMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJppc9ACJKXYqiQFVV2Gw2WK1WBINBOBwOzMzMyB7agsIQE9E1qaoKg8EAk8mEbdu24Rvf+AYGBwfxs5/9DE1NTbKHt6Dw0AQRfU50T1in08FoNKK2thYPPfQQ7r33XthsNiiKInuICwr3iCntRaMBAJqmAQCWLl2KNWvWQFVVNDY24vLlyzKHmDaysrKwdu1aVFVVxUJsMpmwcuVK6HQ6WCwWPPTQQygpKbnm5zscDpw9exZut/sOjzy9McSU9nQ6HfR6PYQQiEQiEEJg3bp1+Lu/+zvo9Xq88MIL6OnpiUWari8/Px/f+ta3sG3bttgvN1VVkZOTA4PBgNLSUnz3u9+F3++/5uefPn0aQ0NDDPE8McQpQq/XIysrCzqdDrOzs/D7/RBCyB5WylMUBZmZmcjNzQUAhMNhCCFgt9tRWloKg8GAgoICWK3WWIiFEAgEAteNyWKm1+thtVpRXl5+3fttNtt1P7+4uBh2ux0Wi4VzPA8McYqorKzE9u3bUVZWhv/7v//DwYMHEQgEZA8r5en1ejQ0NODhhx+O7RULIbB8+XLk5uZCVVU88sgjqKmpiYVY0zQcO3YMBw8exOzsrOSfYGGprq7GM888g9HRURw9ehSHDh3iHMeBIU4RxcXF2L59O1avXo1QKIQjR44wxHHQ6/VYt24dvvOd78BkMsVuVxQl9oLSV7/6VdTX18fuC4VCCAQC+N///V9GIsFKS0vx6KOPYnZ2Fj6fD0ePHuUcx+GWQ5yRkYGKigrYbDa43W709/dzwuNkt9uxZMkSGAyG2G2rVq1CXl4e9Ho9ysrKcM8998Dr9cbun5iY4BxfR/TFuugxzWvdf/Wr/Hq9HqWlpdiwYQPcbjcGBgYwMTFxp4a74EVf5OPKivjFHWKdThf7sxACNpsNTz75JDZv3oyTJ0/i5ZdfxuDgYFIGudCsX78ezz77LAoLC2O35ebmoqKiAnq9Hps2bUJtbS0ikUjs/uPHj+Oll17C0NCQjCEvKKqq4qGHHsKKFSswMDCAl156CcePH5c9LFrE4g6xXv+HDxVCICsrC8uWLcOGDRswNjYWe6FJ0zS+yHQTdrsda9euRWlp6TXvLy4uRnFx8ZzbhoeH5zz1plunKEpsji0WCywWi+whLTjcG56fuEP8zDPPxP4shIDFYsGyZcug0+lQXV2NnTt3Ynh4GGfOnEFjY+OcvTkiWnw+e0iIri/uEP/oRz+a8/+qqiIzMxOqqmLFihWoqqrC9PQ0XnzxRTQ3NzPERATgkyDzWfKNxR1is9l83fsMBgMMBgP0ej2Ki4tRUVGBmZkZuN1uvriUIJmZmSgpKUEgEMD09DQ8Ho/sIS0Ier0edrsdFRUV8Pl8mJyc5E5EguTm5qKsrAxTU1OYnJzkmuIbSOjyNaPRiC1btmDJkiXo7e3FG2+8gba2tkR+i0Xri1/8Ip577jmMj4/j3XffxcGDBxmMBLBardi5cycaGhpw5swZ7N27F06nU/aw0p5Op0N9fT1sNhuGhoawd+9enD9/XvawUlZCQ6zX6/GlL30Jq1evRnNzMw4dOsQQJ0h5eTnKy8vh8XjQ0dGBQ4cOMcQJkJOTgwceeADAJ0sy9+/fzxAngKqqWLlyJerq6tDd3Y3jx48zxDeQ0BBHIhF0d3ejp6cHly9f5trMBHI4HOjo6IDL5eJ5ExLI7/ejvb0dDocDjY2NfPqcIEII9Pb2oqurC4ODgxgbG5M9pJSW0BAHg0H89re/xWuvvQav18vJT6DW1lY8//zz6O/vx/j4OPeGE8TlcuGXv/wlPvjgA3g8Hu48JEg4HMaxY8fw6quvYnJyEg6HQ/aQUlrcIf7s220VRYHBYICqqtA0DaFQCDMzMxgZGUFnZyeCwWDCB7uYeTwe9PT0oLu7W/ZQFpRgMIjBwUFcunSJr+wn2MTEBLq6uuD1evkM7ibiDvELL7wQWxeo1+uRn5+PTZs2oa6uDl1dXXj//fcxOjqKU6dOcW+N0g4jnHjREzBxbm8u7hD/27/9W2wv2GAwoKKiAmVlZVi+fDm6urqwa9cuXL58GeFwmCFOAj6YKd0wxPGLO8R+vx+KoiAUCsFgMGBqagr9/f24ePEi+vr64PF4uGY4TlNTU+jq6sLU1BSAP5xTt6ioCBkZGXC73XA6nXPeLj44OMjDPQnkcrngdDoxMDDANdlJwPjOT9whDofDAD5ZGREKhTA8PIzXXnsNBw4cgNPp5JKfeTh79iz+6Z/+CSaTKXamqhUrVuC73/0uqqqqcOzYMezduxc+ny/2gB4bG8P4+LjkkS8MkUgEx44dw549e+ByudDZ2Sl7SLTIxR3iaBAikUgsxryS660ZGRnByMhI7Hi7qqrwer3YsWMHNE1DX18fjh49iunpae5ZJIGmaejt7cWRI0fmnGqUSBaeGF6y6OGH0dFRHDx4EK2trTh37hyCwSAjHAchBMbGxtDW1oacnByYzWZkZGTAaDTCZDIhFAqho6MD/f39CIfDCAQCCAaDOH/+fOxZHiWO0+lEU1MTxsfH0dzczDmOE0MsUfRilwDQ09ODl19+GQaDAV6vl1fniFMkEkFfXx9OnDgBi8WC5cuXw2azIS8vD0ajEX6/H4cPH8aBAwfg8/kwPj6OYDAIr9fL1zSSoL+/H7t27UJ7e3tsrunmGOIUEQwGeZz9FgghYm8eCoVCsQtbhkIhKIoCr9cLh8OB4eFh+P1+jI+PM8BJFH0cj4yMwO/3c/1wnBhiSmuapqG7uxuBQAAZGRk4deoUTCYTMjIykJmZGXstY3x8HOFwmE+Vk0zTNASDQczOznKu54EhprSmaRoGBwdjl5C61onIuZb1zhFCIBgMIhgM8v0E88AQ04IQDS2DS+lIEXzkEhFJde3rjxMR0R3DEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExHJJuIEgBs3btzSclNVNbbpdDrxjW98Q7S3twun0yl+8IMfiIyMjNh9Op1OqKqasO8dD16hgyhNqKoKRVEghOBFOeMQna/oBmDOf6ObTqeDTqeLXd1FURRomhb7WE3Tkn7lF4aYKA3Y7XZ85StfQVlZGS5evIjTp0/D7/fLHlbKKi0tRX19PQoLCwFgTowBYNWqVbBYLMjIyMD999+PSCQyJ7bRX3ZerxenTp1CR0dHUsfLEBOlgeLiYjz55JO499578eabb6K5uZkhvoHKykp873vfw5e+9KXPRRgA9Ho9srKyoCgKtmzZgoaGhjn3a5oGTdMwOjqKf/7nf2aIKT2pqor8/HxkZ2djdnYWk5OTCIVCsoeVFkwmEywWC/T6P/zzLC0thc1mg8Vigd1uR3l5ObKysmL3BwKBRT/HVz/mSkpKYvN1rSt7Xy0zMxOZmZlzbovuEYdCIZSUlKCiogKBQAButxvhcDjhY4/74qE3+2GIrpaXl4c/+7M/Q0NDA9rb27F7924MDAzIHlZaWLduHf7iL/4CRUVFsdssFgvWrFmDgoIC9Pb2orm5GYFAIHb/xYsXsXv3bly5ckXGkFNCTk4OHn30UWzevBnFxcVYu3YtrFbrLX89IQQCgQAuXLiA3t5etLS04I033sDg4OC8v048H8RVE9wSvhUUFIhdu3aJSCQijh49KlavXi19TOmyPfLII+Ly5cvx/tMUQghx5MgRsWrVKuljl7lZrVbx05/+VITD4XnNXbwOHz4svvjFL857XPHgoYlPKYqCJUuWoKamBsFgEB0dHRgbG5M9rLSQm5uLFStWwGazxW7Lz89HRUUFFEWBzWbD/fffj7Kystj9MzMznOOrmEwm1NXVoaSkBOvWrZtz2CEeNpst9mJed3c3enp6Ft3Kis+ukEgnDPGndDodNm3ahL/6q7/CxMQEXnjhBUYiTiUlJXjmmWewcePG2G06nQ4FBQVQFAU1NTV47rnnMDs7G3ua1t/fj+eff55z/Cmr1YqnnnoKW7duRXZ2Nux2+7w+f9myZXjuuefg8Xjw2muv4dVXX51z6IJS26IPsaqqMBgMyMjIQHFxMWpra+F0OmG325GZmYlIJIJQKJT0dYTpzGg0oqKiAsuXL7/m/VlZWaiqqppzm06nQ05Ozh0YXXowGAwoLS297hzeTHSOZ2dnYbPZoKp802w6WfQhrqiowNe//nWUlZVh48aNMJlMsFqt2L59O1atWoW2tjYcOnQIk5OTsoe6oKTj08dk45yktmT+/TDEFRV46qmncPfdd0On00Gv1yMjIwPf/OY3EYlEsH//fpw+fZohprSwWGN+p44NR79Pop8hL8oQ63Q6FBYWwmKxoLq6Gnl5ecjIyIjdrygKDAYDDAYDbDYbamtrkZmZCafTCZfLJXHkqUNRFBQUFMBqtaKmpgbZ2dnz+nyj0Yjy8nKsXLkS09PTcDgci3INrNVqRUFBASoqKpCXl3fbX09RFBQWFqKurg7T09MYHR2F1+tNwEhTV35+PoqKimC32+NaN3yrsrOzUV1djXA4DJfLhfHx8YQFeVGuI87Ly8O3v/1tbN26FTabDXV1dcjNzb3mxzocDly6dAkTExPYs2cP9u/fj0gkcodHnHoyMjLw+OOPY/v27bBarairq5vXmk2fz4dLly5hbGwMH374IXbt2rXoXrjT6XR45JFHsHPnTthsNixfvnzO2uFbIYRAb28venp6MDAwgJ/97Gf4+OOPEzTi1PTggw/iO9/5DoqKilBbW4uKioqkfB+Xy4W2tjZMTEzg17/+Nfbt24dgMHjTz4snsYtyj9hoNGLlypX42te+FjsxyPUUFRWhsLAQ09PTOHHixIL6hXQ7dDodli1bhoceeggZGRnznpesrCysWbMGQghMT0/DZDIlaaSpK7pkctOmTcjPz0/IY0tRFFRXV2Pp0qXo7OzEu+++m4CRpraSkhLU19ejpKQkqS9SWq1W3HvvvfD7/WhqaoJOp0vY116UIZ6dncXp06eRmZmJoqIibNiwARaL5ZofOzAwgHPnzmF8fBwdHR2Lbm3m9YTDYTQ1NeGtt95CQUEB1q9fj+Li4rg/3+Px4Ny5cxgcHMTHH3+MmZmZJI42NWmaho6ODrz99tsoKCjAunXrUFlZedtfs62tDS0tLRgaGsLQ0FCCRpu6ent78e6776KoqAhr1qxBbW1tUr7P6Ogozpw5g7GxMVy4cCGxz4zjfVcJUuCdM4naVFUVZrNZlJWViT/5kz8RLS0t1/25f/vb34ovf/nLoqSkROTk5Egfe6psiqKIvLw8UVpaKh588EHx4YcfxvtQEkII0dvbK5566ilRVlYmrFar0Ol00n8mGVtOTo4oKSkR99xzjzhw4MC85vBaZmdnxUsvvSRqampEUVGRMJlM0n/GZG9ZWVmiuLhYrFq1SuzevVtomnbb83gtJ06cEJs2bRJlZWUiLy9PKIoS1/jisSj3iDVNw+TkJCYnJ+FwOOByueB2u2EymZCZmQlN0+Dz+RAMBuF0OjEyMoKRkRHZw04p4tNDCtPT07BarXEdK7ta9AWPxbDHdiNerxderxdGozFhb8DweDwYGRlZNM8yfD4ffD4fQqEQZmZmIIRIyiHE2dlZOJ1ODA8Pc9VEovX19eGVV15BcXExNm3ahC1btsDr9eI3v/kNLly4gO7ubrjdbtnDJIqL+PSsYYkOBX1CCJGUuV30IR4eHsavfvUrGI1GmEwmbN68GVNTUzh48CDee++9pE38Ysc5/TzOSWpL5t/Pog8x8MmhinA4HHvhyO12w+Vy8YW5OPn9frS1tcXO6Rpdh11RURFbcdLX1wefzxd7MA8ODvKZxlVmZ2dx6dIlnDp1ClarFUuWLJnXSpLp6Wn09vZiamoKg4ODfOymGYb4U5FIBEePHkVHRwdCoRDPnTsPw8PD+I//+I85b0iwWCz4/ve/j23btqGrqwv//u//jq6urtj9gUCAc3yViYkJvP766zhw4ADq6+vxt3/7tygvL4/78zs7O/Hiiy+iq6sLo6Oj8z5mT3IxxJ8SQvBFuVvk9/s/dymZgoICOBwOhMNhuN1utLS0oLm5WdIIU18wGMTly5cBfLJ2fWZmBuFweM7FLqNrZKPHga8WnePW1tY7O/AUEp2XcDgcuyBoIkQvm/TZ69olEkNMSeH3+3Hs2DH4fD5cvnwZ4+PjsoeUNnp7e7Fnzx7Y7fZYiAsKCuZcPPTUqVNzrlnX2dm56N9+HwgEcPz4cQSDQZSXl8+5eOit8vl8+N3vfof29nZ0dHRgYmIiQaP9jHjX0CEF1gtyS59NURSRlZUlzGazyMnJWbTrhG9lMxgMIi8vT5jNZmGxWITFYhH19fXixIkTIhwOi9dff13U1NQIi8UizGZzbI5VVZU+dpmboigiMzNTmM1m8fWvf100NTXFm7frGh0dFd///veFxWK55TmOB/eIKSmEELH1nTQ/oVBozgmQFEWBy+XCyMgIrly5gtHRUbjdbr7Y+RlCCPj9fvj9fjidTgwNDX3u/CfRZxgmkwlmsxmqqmJ6ejp2YiTxmUMPTqcTTqcz6XO9KE/6Q5RuzGYz1qxZg8LCQnR3d6OlpQWzs7Oyh5WyCgoKbnjx0PXr1+Oxxx5Dbm4u3nvvPRw+fPiaV2eOnleir6/vlscSV2Lj3UVHCjz14MaNG7fb2RRFEaqqij/+4z8W7e3twul0ih/84AfCaDTG/Zbl+W48NEFE9BlCCDgcDpw4cQL5+fno6+uT/m5EHpogokUnJycHdrsder0+dq6ZZIknsQwxEVESxZNYXuqViEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDKGmIhIMoaYiEgyhpiISDJ9vB8ohEjmOIiIFi3uERMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRScYQExFJxhATEUnGEBMRySbiBIAbN27cpG6KoghVVYWqqkKn0wmdTid27twp+vr6xNDQkPje974n9Hr9nPtVVZU65njEfYUOIkocVVWhKAo0TePVb+IQna/oBiD2Z1VVY/+vqip0Ol3s4wFA0zQoigIhRGxLNQwx0R1WXFyM+vp62Gw2tLS04OOPP0YwGJQ9rJRVWVmJ+vp6WK3WORGO/nft2rXIzc2FXq/Hpk2bkJmZOSe2QghomoapqSmcPHkSPT09Un6OG2GIie6w8vJyPP3007jrrruwa9cuXLhwgSG+gS984Qt49tlnUVtbO2ePOMpoNMJkMkFRFGzbtg0PP/zwnPs1TYOmaejv74fL5WKIKT3pdDqYzWZkZmYiEAhgcnIS4XBY9rDSQlZWFvLz86HX/+GfWllZGWw2GywWCwoLC1FWVoaZmZnY/X6/H263G5FIRMaQU4JOp0N+fj6ysrLmzNdnI/xZmZmZyMzMnHNbdI/Y6/WipKQEFRUV8Pl8mJycTJk5VkScB0xuNgG0cNlsNvz5n/85Nm7ciPPnz2PPnj0YHR2VPay0cP/99+Oxxx6DzWaL3Wa327F27VqYzWZ0dXWhtbUVoVAodv+5c+fwxhtvYGxsTMaQU4LFYsG3vvUt3HfffSgtLcXatWuRl5d3y19PCAGv14umpiYMDg7izJkz2Lt37x2Z47gSy1UT3G62VVZWin379gkhhDhw4ICoqamRPqZ02R5//HFx5cqVeP+ZCSGE2L9/v/jCF74gfewyt9LSUvHGG28ITdPmNXfx+s1vfiOqq6vvyM8SjwV5aEJVVVRXV2Pp0qXw+Xxob2/HxMSE7GGlBbPZjLq6OlgslthtBQUFKCkpAQAUFhaioaEBy5Yti93v8XjQ1tbGOf5UdnY2VqxYgYKCAtx9992fe6p8M0VFRaivr0d1dTW6u7vR29ubkq/0J9NnX5Rb6BZkiI1GI7Zu3Ypvf/vbGBgYwPPPP4/Tp0/LHlZaqKqqwg9/+EOsXr06dpter0dhYSEAYOXKlfj7v/97BIPBWBw6Ozvx4x//GB999JGUMaeaoqIi/OVf/iW+8pWvIC8vD/n5+fP6/FWrVuEf/uEfMDU1hVdeeQW//OUv5xy6WCyiy9IWgwUVYlVVYTAYkJWVhZKSEtTW1kKv18NisSAzMxPhcHhRPqDnw2QyYcmSJVi+fPk178/JyUFOTs6c24LBILKysu7E8NKC0WhEeXn5defwZqJz7PV6YbPZFs1e4WK2oEJcU1ODrVu3ori4GPfffz+MRiMKCwvx+OOPY+PGjWhqasIHH3wAr9cre6gLCkPxeZyT1JdKf0cLKsTV1dV4+umnY3vCOp0ORUVFeOyxxxCJRLBnzx78/ve/Z4iJUti11gon43tE/5sKx9/TPsR6vR5FRUXIz89HVVUVcnNzkZGREbtfURQYDAYYDAYUFBRg+fLlyM/Px9jYGCYnJ+UNPIWoqorCwkKYzWYsXbp03i8uZWRkoKqqCitXroTb7cbY2FjKrM+8UxRFgc1mg91uR01NDbKzs2/7a+p0OhQWFmLFihXweDwYHR2Fz+dLwGhTV3RtdWlp6W0tV7uZnJwc1NTUwGAwwOl0wuVyJe17xSPt1xHb7XY8/fTTaGhoQEFBAerq6q57vHJ4eBidnZ1wOp34xS9+gffffz8lfhvKlp2djSeeeALbtm2D1WpFXV3dvF5g8nq96OjogMvlwqFDh/CLX/xi0f2SMxgM2LFjB3bs2BGbQ7vdfltfMxKJoKenB319fbh8+TJ++tOforW1NUEjTj2KouCP/uiP8MQTT8R2mqKrdRLN4XDEHrNvvvkm9u/fn7Sdh3gak/Z7xBkZGbjrrrtib2u80S+M0tJSlJSUwOl04vDhw3dqiCnPYDCgrq4OX/va16DT6eb9SzcnJwfr1q2DEAKDg4MwGo1JGmnqii6Z3Lx5M7KzsxOy46LT6VBTU4OamhrY7Xbs27cvASNNXYqioLy8HA0NDbBarUldNVFYWAibzYbp6WmcPHlS+o5m2ofY5/Ph5MmT0DQNZWVlWL9+PXJzc6/5sT09PTh//jzGxsZw+fLlOzzS1BUMBnH27Fm8+eabKCwsxIYNG+a8E+xmJicncfbsWYyMjOD06dMIBAJJHG1q0jQNLS0t2LdvHwoKCrB+/XqUlpbe1tcMh8NoaWlBe3s7enp64HA4EjTa1CSEQFdXF9555x0UFBRg3bp1WLp0aVK+V/TddePj4+jo6JD/zDjed6IgBd5tc61Np9MJi8UiysrKxBNPPCG6u7uv+zO8/fbb4u677xYlJSUiKytL+thTZVMUReTn54vS0lLxyCOPiMbGxngfFkIIIdrb28WOHTtEWVmZMJvN0s//KmvLzc0VpaWl4qtf/ao4duzYvObwWmZmZsSPf/xjUVVVJQoLC4XRaJT+MyZ7y87OFiUlJWLdunXi3Xffve05vJ73339f3HfffaKsrEzk5uYm9WeKR9rvEUciEbjdbrjdboyMjMDlcsXWDZtMJkQiEfh8PoRCITidTgwPDy/q9/BfixACU1NTmJqawtjY2LzPBBad26GhoSSNMD14PB54PB7k5eVhdnb2tr+epmmYnp7G8PDwojk728zMDGZmZqCqKvx+f9K+TyAQgMPhwPDwsPy9YSyAQxNX6+zsxE9+8hMUFhbi4YcfxoMPPgin04lf//rXuHTpEjo6Orh0LQlS4YFMNB/Rx2yqPHYXVIj7+/tx5coVZGVlwWq1oqGhAS6XCwcOHMCRI0dS9uz8tPAk6nHGx2zypNK8LqgQA588nQuFQhgYGMBHH32EK1euwO12Q9M02UNLC16vFy0tLbGlPIqiwGg0YsmSJbDZbHC73ejv70cgEIg9kLu7uzE9PS1z2CnF7/ejo6MD+fn5KCgoQGVl5bxWkrjdbvT19cHtdqfMU2dKrgUXYuCTY5b/8z//g8bGRgQCAQwMDMgeUtro6+vDiy++OOd8EsXFxfibv/kbPPjgg7h48SJ+8pOfzJlTn8/HOb6Kw+HAq6++irfeegtbtmzBX//1X8dOmhSPlpYW/L//9/8wMDCA4eHhRffmmKjFtPO0IEOsaRqGhoYW/YtHt8Lr9aKtrW3ObZWVlXA6nYhEInC5XGhqauLyvxsIBAK4dOkSgE8u8+P3+xEOh+e8rTa6RlZ8evWIq7lcLly4cCElL+lzp0QPyYTDYaiqmrA1xdHLJkUikZQK/YIMMSWWx+PBoUOHMDY2hosXL2Jqakr2kNJGZ2cnXn/9deTn58dCXFJSgvr6etjtdjQ3N+Ojjz6asyqipaVl0c/xzMwMPvjgA0xMTKCqqgr19fVzzpF9K6anp3HixAl0d3ejpaUFHo8nQaNNgHjX3SEF1hhyk7MpiiKysrKE2WwW2dnZi3ad8K1sBoNB5OXlCbPZLCwWi7BYLGLLli3i7NmzYnZ2Vrz88suisrJSWCwWYTabOcefblc/5rZv3y4uXboU9xrh6+nr6xM7d+6843McD+4R000JIeDz+Rb8CWeSIRQKzTkHtqIocLlcGB4ehtlshsPhgNvtTq29sxRw9WNufHwcQ0NDnzsZVfQZRvQCrYqiYGpqKnYhVvGZFzmHhobgdDpT8jwoaX/SH6J0c/XFQzs7O3Hx4kVesOAGSkpKcPfdd8NsNn/uPlVVcd9992HHjh3Q6/V45513cPz48Wse//V6vWhsbMTg4OAdGPUfxJNY7hET3WHj4+M86dQ8jIyMYGRk5HO3R89bHD1rm9FoxJkzZ/Bf//Vf0DQtrZb9McRElLaEEBgaGsLx48eh0+lw5cqVtHwTDA9NEFFay8vLg91uh6IoGB8fT7kVJ/EkliEmIkqieBK7eK5XTUSUohhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJ9PF+oBAimeMgIlq0uEdMRCQZQ0xEJBlDTEQkGUNMRCQZQ0xEJBlDTEQkGUNMRCQZQ0xEJBlDTEQkGUNMRCQZQ0xEJBlDTEQkGUNMRCQZQ0xEJBlDTEQkWdznI1YUJZnjICJakOI5lzv3iImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCRjiImIJGOIiYgkY4iJiCTTyx4AUSIoigIAEEJIHkn6U1U1Np83IoSApml3YEQLH0NMaU1RFNjtdthsNgSDQTgcDszMzMgeVtrKycnBAw88gNra2pt+bFtbG37/+9/D5/PdgZEtbAwxpTVVVVFUVISVK1fC4/HA7/fD5/Nxz/gW5efn49FHH8U3v/nNG36cEAJ79+7FhQsXGOIEYIglUBQFOp0OABCJRBiN26AoCnJyclBcXIzc3Fw4nU4YDIbYnEYiEUxNTXEvOU6qqiI7Oxtms/mGHyeEQGFhIcrKymAwGDjHt4khlsBkMiE/Px8A4PF4+AC+DTqdDnfddRe2b98ORVEwMTEBv98fC7HH48Hbb7+No0eP8hdeAimKgg0bNuBHP/oRxsbGsG/fPhw/fpxzfIsYYgkMBgNycnKgqipmZ2f5VPo2qKqK8vJy3HPPPTCZTJ+7f3x8HOfOncPRo0cljG5hW7p0KZYuXYqxsTGcO3cOx48flz2ktBV3iKurq6FpGjweD9xuN18tvQ2FhYXYuHEjTCYTpqam4PV6AQCapkEIgf7+fnR3dyMcDkseafozGo1YtWoVtm7diomJCbS3t2N6elr2sBaUeFZY0I3FHeLNmzcjEomgvb0d58+fRzAYTOa4FrRVq1bh2WefRVFREcLhcCzA4XAYoVAIb775Jl555RWGOAGys7OxY8cOPPzwwzhz5gz+5V/+BW1tbbKHRTRH3CEuKiqCpmkYHR1FVlYWVFVFKBRCJBJJ5vgWpJycHFRWVqK0tDR229UhLiwsjL2YR7dHp9OhqKgIRUVFcDgc1zx8QSRb3CFuaGgAAKxYsQIPPPAAxsfHcfjwYVy8eDFZY1tUoispNE2DqvINj5R+FEXhax23aN4hvvo45sDAAEOcQKqqQqfTMcSUdhRFiW2M8fzFHWK9fu6HRp9eRxfSj46OIhQKJXyAi42iKLBarairq8PExAQcDgc8Ho/sYS0I2dnZqK6uhs/nw8TEBMbHx/micwLodDqUlJSgrq4OXq8Xo6OjmJ2dlT2stHLLy9esViuefPJJbNmyBadPn8bPf/5zDA8PJ3Jsi5KqqmhoaEB5eTmGhoawa9cunDx5UvawFoTq6mr88Ic/hMvlwnvvvYe9e/fC7/fLHlbay8nJwZ/+6Z/iy1/+Mpqbm/Hzn/8cPT09soeVVm45xCaTCatXr8Zdd92FQCCArKysRI5r0VIUBUuWLEFlZSX6+vqwf/9+2UNaMCwWCzZu3IhgMIi2tja+IJogRqMRK1asQG1tbWyNPM3PLYfY6/Xi/Pnz6O/vR2NjI58+J4gQAh0dHWhubsbw8DCuXLkie0gLxtjYGM6ePYuxsTE0NjZyeWCCBAIBNDY2oqurC+3t7ZicnJQ9pLRzyyGenJzEW2+9hQMHDsDv92NqaiqR41q0IpEITp48iRdffBFut5vzmkC9vb14+eWX0dLSAq/Xy+OYCTIzM4P9+/fjrbfeQiAQYIhvQdwhFkJACIHZ2VkEAgG4XC6Mjo5iaGgomeNblLxeL4aHh/kOsAQLBoNwOp18zCaYpmmYnJzE8PBwbFUVzU/cIfZ4PNA0DadOncKRI0fgdDrR2tqazLEtOtETbfOBnBzRnQlKDs7vrYs7xD6fD+FwGOfOncN//ud/Ynp6mkt/Eiz6QOaDmdINH7e3J+4Qnz17FpqmYWBgAMFgkBG+DdE931AohMHBQYyNjc0530R/fz/fOp4g4XAYAwMDGBsbw8WLF3nKUUpJcYf4H//xHwEATqcTgUAgaQNaDDRNg6ZpmJ6exq9+9Su89957sRPEa5oGh8PBOU6QmZkZvP3229i/fz+mpqa4CiUJuCd8++IOcVNTUxKHsbhE93z9fj96enrw8ccf80odCfLZY+yBQCA2x5zfxBJCIBKJIBKJ8BnybeKJ4SXo6urC7t27odfr0draygfxbYoen3S5XPjwww9je71CCHi9Xp4PJUna29vxu9/9Dg6HAxcvXuQvutvAEEvQ3NyMrq4uKIoCv9/PECeAEAIOhwO7d+/GiRMn5qxAufrSSZQYQgg0NTXhX//1X+FwOPhW8dvEEEsQCoV4gqQEEUJgamoKg4ODGBoagsvlgtvt5qv4tyj6C0zTNMzOzmJqair2DsSr51MIgZGREUxMTPBNRwnAEFNaC4VC+OCDDzA4OAiv14vOzk5G+DYIIRAMBuHz+dDa2op33nkHDofjc6t4hBDo7e2NXeaLbg9DTGktEomgtbWVby5KECEEQqEQgsEgBgYG8N///d/o7u6O7SVTcjDERBQzOzuL1tZW5OXlobGxETMzM3yGcQcoIs4Z5pVaiRY+vV4Pu92OnJwc+Hw+jI+P80LBtymexDLERERJFE9ieXE0IiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIskYYiIiyRhiIiLJGGIiIsniPvsaz75ERJQc3CMmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKSjCEmIpKMISYikowhJiKS7P8D6+vs/P3Fr5MAAAAASUVORK5CYII=", "text/plain": [ "
" ] diff --git a/serket/_src/nn/linear.py b/serket/_src/nn/linear.py index 2539192..2fd28ca 100644 --- a/serket/_src/nn/linear.py +++ b/serket/_src/nn/linear.py @@ -61,12 +61,19 @@ def general_linear( out = "".join(str(axis) for axis in range(input.ndim) if axis not in in_axis) out_axis = out_axis if out_axis >= 0 else out_axis + len(out) + 1 out = out[:out_axis] + "F" + out[out_axis:] - result = jnp.einsum(f"{lhs},{rhs}->{out}", input, weight) + + try: + einsum = f"{lhs},{rhs}->{out}" + result = jnp.einsum(einsum, input, weight) + except ValueError as error: + raise ValueError(f"{einsum=}\n{input.shape=}\n{weight.shape=}\n{error=}") if bias is None: return result - broadcast_shape = list(range(result.ndim)) - del broadcast_shape[out_axis] + + with jax.ensure_compile_time_eval(): + broadcast_shape = list(range(result.ndim)) + del broadcast_shape[out_axis] bias = jnp.expand_dims(bias, axis=broadcast_shape) return result + bias @@ -306,6 +313,10 @@ class MLP(sk.TreeClass): >>> _, material_layer = lazy_layer.at['__call__'](input) >>> material_layer.in_linear.in_features (10,) + + Note: + :class:`.MLP` uses ``jax.lax.scan`` to reduce the ``jaxpr`` size. + Leading to faster compilation times and smaller ``jaxpr`` size. """ def __init__( diff --git a/serket/_src/nn/normalization.py b/serket/_src/nn/normalization.py index 03412d3..8d4b7b9 100644 --- a/serket/_src/nn/normalization.py +++ b/serket/_src/nn/normalization.py @@ -211,6 +211,7 @@ class GroupNorm(sk.TreeClass): >>> import serket as sk >>> import jax.numpy as jnp >>> import jax.random as jr + >>> key = jr.PRNGKey(0) >>> layer = sk.nn.GroupNorm(5, groups=1, key=key) >>> input = jnp.ones((5,10)) >>> layer(input).shape @@ -426,13 +427,12 @@ class BatchNorm(sk.TreeClass): .. warning:: Works under - - ``jax.vmap(BatchNorm(...), in_axes=(0, None), out_axes=(0, None))(x, state)`` - - ``jax.vmap(BatchNorm(...), out_axes=(0, None))(x)`` + - ``jax.vmap(BatchNorm(...), in_axes=(0, None), out_axes=(0, None))(input, state)`` otherwise will be a no-op. Training behavior: - - ``output = (x - batch_mean) / sqrt(batch_var + eps)`` + - ``output = (input - batch_mean) / sqrt(batch_var + eps)`` - ``running_mean = momentum * running_mean + (1 - momentum) * batch_mean`` - ``running_var = momentum * running_var + (1 - momentum) * batch_var`` @@ -461,8 +461,9 @@ class BatchNorm(sk.TreeClass): >>> import jax.random as jr >>> bn = sk.nn.BatchNorm(10, key=jr.PRNGKey(0)) >>> state = sk.tree_state(bn) - >>> x = jax.random.uniform(jax.random.PRNGKey(0), shape=(5, 10)) - >>> x, state = jax.vmap(bn, in_axes=(0, None), out_axes=(0, None))(x, state) + >>> key = jr.PRNGKey(0) + >>> input = jr.uniform(key, shape=(5, 10)) + >>> output, state = jax.vmap(bn, in_axes=(0, None), out_axes=(0, None))(input, state) Example: Working with :class:`.BatchNorm` with threading the state. @@ -476,19 +477,19 @@ class BatchNorm(sk.TreeClass): ... k1, k2 = jax.random.split(key) ... self.bn1 = sk.nn.BatchNorm(5, axis=-1, key=k1) ... self.bn2 = sk.nn.BatchNorm(5, axis=-1, key=k2) - ... def __call__(self, x, state): - ... x, bn1 = self.bn1(x, state.bn1) - ... x = x + 1.0 - ... x, bn2 = self.bn2(x, state.bn2) + ... def __call__(self, input, state): + ... input, bn1 = self.bn1(input, state.bn1) + ... input = input + 1.0 + ... input, bn2 = self.bn2(input, state.bn2) ... # update the output state ... state = state.at["bn1"].set(bn1).at["bn2"].set(bn2) - ... return x, state + ... return input, state >>> net: ThreadedBatchNorm = ThreadedBatchNorm(key=jr.PRNGKey(0)) >>> # initialize state as the same structure as tree >>> state: ThreadedBatchNorm = sk.tree_state(net) - >>> x = jnp.linspace(-jnp.pi, jnp.pi, 50 * 20).reshape(20, 10, 5) - >>> for xi in x: - ... out, state = jax.vmap(net, in_axes=(0, None), out_axes=(0, None))(xi, state) + >>> inputs = jnp.linspace(-jnp.pi, jnp.pi, 50 * 20).reshape(20, 10, 5) + >>> for input in inputs: + ... output, state = jax.vmap(net, in_axes=(0, None), out_axes=(0, None))(input, state) Example: Working with :class:`.BatchNorm` without threading the state. @@ -511,28 +512,28 @@ class BatchNorm(sk.TreeClass): ... self.bn1_state = sk.tree_state(self.bn1) ... self.bn2 = sk.nn.BatchNorm(5, axis=-1, key=k2) ... self.bn2_state = sk.tree_state(self.bn2) - ... def _call(self, x): + ... def _call(self, input): ... # this method will raise `AttributeError` if used directly ... # because this method mutates the state ... # instead, use `at["_call"]` to call this method to ... # return the output and updated state in a functional manner - ... x, self.bn1_state = self.bn1(x, self.bn1_state) - ... x = x + 1.0 - ... x, self.bn2_state = self.bn2(x, self.bn2_state) - ... return x - ... def __call__(self, x): - ... return self.at["_call"](x) + ... input, self.bn1_state = self.bn1(input, self.bn1_state) + ... input = input + 1.0 + ... input, self.bn2_state = self.bn2(input, self.bn2_state) + ... return input + ... def __call__(self, input): + ... return self.at["_call"](input) >>> # define a function to mask and unmask the net across `vmap` >>> # this is necessary because `vmap` needs the output to be of inexact - >>> def mask_vmap(net, x): + >>> def mask_vmap(net, input): ... @ft.partial(jax.vmap, out_axes=(0, None)) - ... def forward(x): - ... return sk.tree_mask(net(x)) - ... return sk.tree_unmask(forward(x)) + ... def forward(input): + ... return sk.tree_mask(net(input)) + ... return sk.tree_unmask(forward(input)) >>> net: UnthreadedBatchNorm = UnthreadedBatchNorm(key=jr.PRNGKey(0)) - >>> input = jnp.linspace(-jnp.pi, jnp.pi, 50 * 20).reshape(20, 10, 5) - >>> for xi in input: - ... out, net = mask_vmap(net, xi) + >>> inputs = jnp.linspace(-jnp.pi, jnp.pi, 50 * 20).reshape(20, 10, 5) + >>> for input in inputs: + ... output, net = mask_vmap(net, input) Note: :class:`.BatchNorm` supports lazy initialization, meaning that the @@ -549,8 +550,9 @@ class BatchNorm(sk.TreeClass): >>> key = jr.PRNGKey(0) >>> lazy_layer = sk.nn.BatchNorm(None, key=key) >>> input = jnp.ones((5,10)) - >>> _ , material_layer = lazy_layer.at['__call__'](input) - >>> output, state = jax.vmap(material_layer, out_axes=(0, None))(input) + >>> _ , material_layer = lazy_layer.at["__call__"](input, None) + >>> state = sk.tree_state(material_layer) + >>> output, state = jax.vmap(material_layer, in_axes=(0, None), out_axes=(0, None))(input, state) >>> output.shape (5, 10) @@ -589,11 +591,8 @@ def __init__( @ft.partial(maybe_lazy_call, is_lazy=is_lazy_call, updates=updates) def __call__( - self, - input: jax.Array, - state: BatchNormState | None = None, + self, input: jax.Array, state: BatchNormState ) -> tuple[jax.Array, BatchNormState]: - state = sk.tree_state(self) if state is None else state batchnorm_impl = custom_vmap(lambda x, state: (x, state)) momentum, eps = jax.lax.stop_gradient((self.momentum, self.eps)) @@ -622,13 +621,12 @@ class EvalNorm(sk.TreeClass): .. warning:: Works under - - ``jax.vmap(BatchNorm(...), in_axes=(0, None), out_axes=(0, None))(x, state)`` - - ``jax.vmap(BatchNorm(...), out_axes=(0, None))(x)`` + - ``jax.vmap(BatchNorm(...), in_axes=(0, None), out_axes=(0, None))(input, state)`` otherwise will be a no-op. Evaluation behavior: - - ``output = (x - running_mean) / sqrt(running_var + eps)`` + - ``output = (input - running_mean) / sqrt(running_var + eps)`` Args: in_features: the shape of the input to be normalized. @@ -653,10 +651,10 @@ class EvalNorm(sk.TreeClass): >>> bn = sk.nn.BatchNorm(10, key=jr.PRNGKey(0)) >>> state = sk.tree_state(bn) >>> input = jax.random.uniform(jr.PRNGKey(0), shape=(5, 10)) - >>> output, state = jax.vmap(bn, in_axes=(0, None), out_axes=(0, None))(x, state) + >>> output, state = jax.vmap(bn, in_axes=(0, None), out_axes=(0, None))(input, state) >>> # convert to evaluation mode >>> bn = sk.tree_eval(bn) - >>> output, state = jax.vmap(bn, in_axes=(0, None))(input, state) + >>> output, state = jax.vmap(bn, in_axes=(0, None), out_axes=(0,None))(input, state) Note: If ``axis_name`` is specified, then ``axis_name`` argument must be passed diff --git a/serket/_src/nn/recurrent.py b/serket/_src/nn/recurrent.py index 25c5c9c..c34688b 100644 --- a/serket/_src/nn/recurrent.py +++ b/serket/_src/nn/recurrent.py @@ -16,13 +16,12 @@ import abc import functools as ft -from contextlib import suppress -from typing import Any, Callable import jax import jax.numpy as jnp import jax.random as jr -import jax.tree_util as jtu +from typing_extensions import ParamSpec +from typing import Callable, Any, TypeVar import serket as sk from serket._src.custom_transform import tree_state @@ -48,6 +47,10 @@ validate_spatial_nd, ) +P = ParamSpec("P") +T = TypeVar("T") +S = TypeVar("S") + State = Any """Defines RNN related classes.""" @@ -73,100 +76,11 @@ class RNNState(sk.TreeClass): hidden_state: jax.Array -class RNNCell(sk.TreeClass): - """Abstract class for RNN cells. - - Subclass this class to define a new RNN cell that can be used with :class:`nn.ScanRNN`. - or :func:`nn.scan_rnn`. - - Subclasses must - - Implement ``__call__`` method that accept an input and a state and returns - tuple of output and new state. - - Define state rule using :func:`serket.tree_state` decorator. - - Define ``spatial_ndim`` attribute that specifies the spatial dimension of - the cell. For non-spatial cells (e.g. :class:`.LSTMCell`), set ``spatial_ndim`` to 0, - for 1D cells (e.g. :class:`.ConvLSTM1DCell` ) set it to 1 and so on. - - Note: - :class:`.ScanRNN` and :func:`.scan_rnn` offers a unified interface for - scanning over time steps of RNN cells. Supports forward and backward - scanning, helpful error messages for wrong input shapes and more. - - Example: - Define a simple ``RNN`` cell that matrix multiplies the input with a ones matrix - and adds the result to the hidden state. - - >>> import serket as sk - >>> import jax - >>> import jax.numpy as jnp - >>> class CustomRNNState(sk.TreeClass): - ... def __init__(self, hidden_state: jax.Array): - ... self.hidden_state = hidden_state - - >>> class CustomRNNCell(sk.nn.RNNCell): - ... def __init__(self, in_features: int, hidden_features: int): - ... self.in_features = in_features - ... self.hidden_features = hidden_features - ... self.in_to_hidden = lambda x: x @ jnp.ones((in_features, hidden_features)) - ... def __call__( - ... self, - ... input: jax.Array, - ... state: CustomRNNState | None = None, - ... ) -> CustomRNNState: - ... # if no state is provided, by default it will be initialized with - ... # rule defined using `sk.tree_state.def_state` below when the cell is - ... # wrapped with `sk.nn.ScanRNN`/`sk.nn.scan_rnn` - ... output = self.in_to_hidden(input) - ... state = CustomRNNState(state.hidden_state + output) - ... return output, state - ... # to validate the shape of the input and give more helpful error message - ... # define the shape of the input input. e.g. in case of Non-spatial RNN - ... # spatial_ndim should be 0, otherwise it should be 1 for 1D (e.g. ConvLSTM1D) - ... # 2 for 2D (e.g. ConvLSTM2D) and so on. - ... spatial_ndim: int = 0 - - >>> # initialize the cell with zeros hidden state - >>> @sk.tree_state.def_state(CustomRNNCell) - ... def custom_rnn_state(cell: CustomRNNCell, **_) -> CustomRNNState: - ... zeros = jnp.zeros((cell.hidden_features,)) - ... return CustomRNNState(hidden_state=zeros) - >>> cell = CustomRNNCell(5, 10) - >>> print(repr(sk.tree_state(cell))) - CustomRNNState(hidden_state=f32[10](μ=0.00, σ=0.00, ∈[0.00,0.00])) - >>> inputs = jnp.ones((5, 5)) # 5 time steps, 5 features - >>> # 5 time steps will perform 5 steps of matrix multiplication of - >>> # the running hidden state with ones matrix of shape (5, 10) and - >>> # add the result to the hidden state - >>> print(sk.nn.ScanRNN(cell)(inputs)) - [25. 25. 25. 25. 25. 25. 25. 25. 25. 25.] - - This is equivalent to the following code: - - >>> import jax.numpy as jnp - >>> h = jnp.zeros(10) # 10 hidden features initialized with zeros - >>> inputs = jnp.ones((5, 5)) # 5 time steps, 5 input_features - >>> for i in range(5): # the scanning as a python loop - ... h = h + inputs[i] @ jnp.ones((5, 10)) - >>> print(h) - [25. 25. 25. 25. 25. 25. 25. 25. 25. 25.] - """ - - @abc.abstractmethod - def __call__(self, input: jax.Array, state: State) -> tuple[jax.Array, State]: - ... - - @property - @abc.abstractmethod - def spatial_ndim(self) -> int: - # 0 for non-spatial, 1 for 1D, 2 for 2D, 3 for 3D etc. - ... - - class SimpleRNNState(RNNState): ... -class SimpleRNNCell(RNNCell): +class SimpleRNNCell(sk.TreeClass): """Vanilla RNN cell that defines the update rule for the hidden state Args: @@ -286,7 +200,7 @@ class DenseState(RNNState): ... -class DenseCell(RNNCell): +class DenseCell(sk.TreeClass): """No hidden state cell that applies a dense(Linear+activation) layer to the input Args: @@ -369,8 +283,8 @@ def __call__( ) -> tuple[jax.Array, DenseState]: if not isinstance(state, DenseState): raise TypeError(f"Expected {state=} to be an instance of `DenseState`") - - h = self.act(self.in_to_hidden(input)) + h = self.in_to_hidden(input) + h = self.act(h) return h, DenseState(h) spatial_ndim: int = 0 @@ -381,7 +295,7 @@ class LSTMState(RNNState): cell_state: jax.Array -class LSTMCell(RNNCell): +class LSTMCell(sk.TreeClass): """LSTM cell that defines the update rule for the hidden state and cell state Args: @@ -513,7 +427,7 @@ class GRUState(RNNState): ... -class GRUCell(RNNCell): +class GRUCell(sk.TreeClass): """GRU cell that defines the update rule for the hidden state and cell state Args: @@ -633,7 +547,7 @@ class ConvLSTMNDState(RNNState): cell_state: jax.Array -class ConvLSTMNDCell(RNNCell): +class ConvLSTMNDCell(sk.TreeClass): @ft.partial(maybe_lazy_init, is_lazy=is_lazy_init) def __init__( self, @@ -1064,7 +978,7 @@ class ConvGRUNDState(RNNState): ... -class ConvGRUNDCell(RNNCell): +class ConvGRUNDCell(sk.TreeClass): @ft.partial(maybe_lazy_init, is_lazy=is_lazy_init) def __init__( self, @@ -1471,99 +1385,30 @@ class FFTConvGRU3DCell(ConvGRUNDCell): convolution_layer = FFTConv3D -# Scanning API - - -def materialize_cell(instance, input: jax.Array, state=None, **__) -> RNNCell: - # in case of lazy initialization, we need to materialize the cell - # before it can be passed to the scan function - cell = instance.cell - state = state if state is not None else sk.tree_state(instance, input=input) - state = split_state(state, 2) if instance.backward_cell is not None else [state] - _, cell = cell.at["__call__"](input[0], state[0]) - return cell - - -def materialize_backward_cell(instance, x, state=None, **__) -> RNNCell | None: - if instance.backward_cell is None: - return None - cell = instance.cell - state = state if state is not None else sk.tree_state(instance, input=x) - state = split_state(state, 2) if instance.backward_cell is not None else [state] - _, cell = cell.at["__call__"](x[0], state[-1]) - return cell - - -def is_lazy_init(_, cell, backward_cell=None, **__) -> bool: - lhs = getattr(cell, "in_features", False) is None - rhs = getattr(backward_cell, "in_features", False) is None - return lhs or rhs - - -def is_lazy_call(instance, x, state=None, **_) -> bool: - lhs = getattr(instance.cell, "in_features", False) is None - rhs = getattr(instance.backward_cell, "in_features", False) is None - return lhs or rhs - - -updates = dict(cell=materialize_cell, backward_cell=materialize_backward_cell) - - -def split_state(state: RNNState, splits: int) -> list[RNNState]: - flat_arrays: list[jax.Array] = jtu.tree_leaves(state) - return [type(state)(*x) for x in zip(*(jnp.split(x, splits) for x in flat_arrays))] - - -def concat_state(states: list[RNNState]) -> RNNState: - # undo the split - return ( - states[0] - if len(states) == 1 - else jax.tree_map(lambda *x: jnp.concatenate([*x]), *states) - ) - - -def scan_rnn( - cell: RNNCell, - backward_cell: RNNCell | None, - input: jax.Array, - state: State, - return_sequences: bool = False, - return_state: bool = False, -) -> jax.Array | tuple[jax.Array, State]: - """Scans a RNN cell(s) over a sequence. +def scan_cell(cell, in_axis=0, out_axis=0, reverse=False): + """Scan am RNN cell over a sequence. Args: - cell: the forward RNN cell to scan. - backward_cell: the backward RNN cell to scan. Pass ``None`` for unidirectional RNN. - input: the input sequence. - state: the initial state of the RNN cell. In case of bidirectional RNN, - the forward and backward states are concatenated along the first axis. - return_sequences: whether to return the output for each timestep. Defaults - to ``False``. - return_state: whether to return the final state of the RNN cell(s). Defaults - to ``False``. + cell: the RNN cell to scan. The cell should have the following signature: + `cell(input, state) -> tuple[output, state]` + in_axis: the axis to scan over. Defaults to 0. + out_axis: the axis to move the output to. Defaults to 0. + reverse: whether to scan the sequence in reverse order. Defaults to ``False``. Example: - Unionidirectional RNN: + Unidirectional RNN: >>> import serket as sk >>> import jax >>> import jax.numpy as jnp - - >>> cell = sk.nn.SimpleRNNCell(1, 2, key=jax.random.PRNGKey(0)) + >>> import jax.random as jr + >>> key = jr.PRNGKey(0) + >>> cell = sk.nn.SimpleRNNCell(1, 2, key=key) >>> state = sk.tree_state(cell) - >>> input = jnp.ones([10, 1]) # [time steps, features] - - >>> out = sk.nn.scan_rnn(cell, None, input, state) - >>> print(out.shape) - (2,) - - >>> out = sk.nn.scan_rnn(cell, None, input, state, return_sequences=True) - >>> print(out.shape) + >>> input = jnp.ones([10, 1]) + >>> output, state = sk.nn.scan_cell(cell)(input, state) + >>> print(output.shape) (10, 2) - - >>> out, state = sk.nn.scan_rnn(cell, None, input, state, return_state=True) Example: Bidirectional RNN: @@ -1571,191 +1416,33 @@ def scan_rnn( >>> import serket as sk >>> import jax >>> import jax.numpy as jnp - - >>> cell = sk.nn.SimpleRNNCell(1, 2, key=jax.random.PRNGKey(0)) - >>> back_cell = sk.nn.SimpleRNNCell(1, 2, key=jax.random.PRNGKey(1)) - >>> # concat state of forward and backward cells - >>> concat_state_func = lambda *x: jnp.concatenate([*x]) - >>> state = jax.tree_map(concat_state_func, *sk.tree_state((cell, back_cell))) - >>> input = jnp.ones([10, 1]) # [time steps, features] - - >>> out = sk.nn.scan_rnn(cell, back_cell, input, state) - >>> print(out.shape) - (4,) - - >>> out = sk.nn.scan_rnn(cell, back_cell, input, state, return_sequences=True) - >>> print(out.shape) - (10, 4) - - >>> out, state = sk.nn.scan_rnn(cell, back_cell, input, state, return_state=True) - - Returns: - return the result and state if ``return_state`` is ``True``. otherwise, - return the result. - - Note: - See :class:`.nn.ScanRNN` for a class-based API. - """ - - def accumulate_scan( - cell: RNNCell, - input: jax.Array, - state: State, - reverse: bool = False, - ) -> tuple[jax.Array, State]: - def scan_func(carry, input): - output, state = cell(input, state=carry) - return state, output - - input = jnp.flip(input, axis=0) if reverse else input # flip over time axis - carry, output = jax.lax.scan(scan_func, state, input) - output = jnp.flip(output, axis=-1) if reverse else output - return output, carry - - def unaccumulate_scan( - cell: RNNCell, - input: jax.Array, - state: State, - reverse: bool = False, - ) -> jax.Array: - def scan_func(carry, input): - _, state = cell(input, state=carry) - return state, None - - input = jnp.flip(input, axis=0) if reverse else input - carry, _ = jax.lax.scan(scan_func, state, input) - result = carry.hidden_state - return result, carry - - if backward_cell is None: - scan_func = accumulate_scan if return_sequences else unaccumulate_scan - result, state = scan_func(cell, input, state) - return (result, state) if return_state else result - # bidirectional RNN - lhs_state, rhs_state = split_state(state, splits=2) - scan_func = accumulate_scan if return_sequences else unaccumulate_scan - lhs_result, lhs_state = scan_func(cell, input, lhs_state, False) - rhs_result, rhs_state = scan_func(backward_cell, input, rhs_state, True) - concat_axis = int(return_sequences) - result = jnp.concatenate((lhs_result, rhs_result), axis=concat_axis) - state = concat_state((lhs_state, rhs_state)) - return (result, state) if return_state else result - - -def check_cells(*cells: Any) -> None: - """Checks that the cells are compatible with each other.""" - cell0, *cells = cells - for cell in cells: - if not isinstance(cell, RNNCell): - raise TypeError(f"{cell=} to be an instance of `RNNCell`.") - with suppress(AttributeError): - # if the user has not specified the in_features, we cannot check - # that the cells are compatible - if cell0.in_features != cell.in_features: - raise ValueError(f"{cell0.in_features=} != {cell.in_features=}") - with suppress(AttributeError): - # if the user has not specified the hidden_features, we cannot check - # that the cells are compatible - if cell0.hidden_features != cell.hidden_features: - raise ValueError(f"{cell0.hidden_features=} != {cell.hidden_features=}") - - -class ScanRNN(sk.TreeClass): - """Scans RNN cell over a sequence. - - Args: - cell: the RNN cell to scan. - backward_cell: (optional) the backward RNN cell to scan in case of bidirectional RNN. - return_sequences: whether to return the output for each timestep. - return_state: whether to return the final state of the RNN cell(s). - - Example: - >>> import jax.numpy as jnp - >>> import serket as sk >>> import jax.random as jr - >>> # 10-dimensional input, 20-dimensional hidden state - >>> cell = sk.nn.SimpleRNNCell(10, 20, key=jr.PRNGKey(0)) - >>> rnn = sk.nn.ScanRNN(cell, return_state=True) - >>> input = jnp.ones((5, 10)) # 5 timesteps, 10 features - >>> output, state = rnn(input) + >>> k1, k2 = jr.split(jr.PRNGKey(0)) + >>> cell1 = sk.nn.SimpleRNNCell(1, 2, key=k1) + >>> cell2 = sk.nn.SimpleRNNCell(1, 2, key=k2) + >>> state1 = sk.tree_state(cell1) + >>> state2 = sk.tree_state(cell2) + >>> input = jnp.ones([10, 1]) + >>> output1, state1 = sk.nn.scan_cell(cell1)(input, state1) + >>> output2, state2 = sk.nn.scan_cell(cell2, reverse=True)(input, state2) + >>> output = jnp.concatenate((output1, output2), axis=1) >>> print(output.shape) - (20,) - - Example: - >>> import jax.numpy as jnp - >>> import serket as sk - >>> import jax.random as jr - >>> cell = sk.nn.SimpleRNNCell(10, 20, key=jr.PRNGKey(0)) - >>> rnn = sk.nn.ScanRNN(cell, return_sequences=True, return_state=True) - >>> input = jnp.ones((5, 10)) # 5 timesteps, 10 features - >>> output, state = rnn(input) # 5 timesteps, 20 features - >>> output.shape - (5, 20) + (10, 4) """ - @ft.partial(maybe_lazy_init, is_lazy=is_lazy_init) - def __init__( - self, - cell: RNNCell, - backward_cell: RNNCell | None = None, - *, - return_sequences: bool = False, - return_state: bool = False, - ): - if not isinstance(cell, RNNCell): - raise TypeError(f"Expected {cell=} to be an instance of `RNNCell`.") - - if backward_cell is not None: - check_cells(cell, backward_cell) + def scan_func(state, input): + output, state = cell(input, state) + return state, output - self.cell = cell - self.backward_cell = backward_cell - self.return_sequences = return_sequences - self.return_state = return_state + def wrapper(input: T, state: S) -> tuple[T, S]: + # push the scan axis to the front + input = jnp.moveaxis(input, in_axis, 0) + state, output = jax.lax.scan(scan_func, state, input, reverse=reverse) + # move the output axis to the desired location + output = jnp.moveaxis(output, 0, out_axis) + return output, state - @ft.partial(maybe_lazy_call, is_lazy=is_lazy_call, updates=updates) - def __call__( - self, - input: jax.Array, - state: State | None = None, - ) -> jax.Array | tuple[jax.Array, State]: - """Scans the RNN cell over a sequence. - - Args: - input: the input sequence. - state: the initial state. if None, state is initialized by the rule - defined using :func:`.tree_state`. - - Returns: - return the result and state if ``return_state`` is True. otherwise, - return only the result. - """ - - if input.ndim != self.cell.spatial_ndim + 2: - raise ValueError( - f"Expected input to have {(self.cell.spatial_ndim + 2)=} dimensions corresponds to " - f"(timesteps, in_features, {', '.join(['...']*self.cell.spatial_ndim)})." - f"\nGot {input.ndim=} and {input.shape=}." - ) - - with suppress(AttributeError): - # if the user has not specified the in_features, we cannot check - # that the cells are compatible - if self.cell.in_features != input.shape[1]: - raise ValueError( - f"Expected input to have shape (timesteps, {self.cell.in_features}, " - f"{', '.join(['...']*self.cell.spatial_ndim)})." - f"\nGot {input.shape[1]=} and {self.cell.in_features=}." - ) - - return scan_rnn( - self.cell, - self.backward_cell, - input, - tree_state(self, input=input) if state is None else state, - self.return_sequences, - self.return_state, - ) + return wrapper # register state handlers @@ -1782,7 +1469,7 @@ def _(cell: GRUCell) -> GRUState: return GRUState(jnp.zeros([cell.hidden_features])) -def _check_rnn_cell_tree_state_input(cell: RNNCell, input): +def _check_rnn_cell_tree_state_input(cell, input): if not (hasattr(input, "ndim") and hasattr(input, "shape")): raise TypeError( f"Expected {input=} to have `ndim` and `shape` attributes." @@ -1792,9 +1479,9 @@ def _check_rnn_cell_tree_state_input(cell: RNNCell, input): if input.ndim != cell.spatial_ndim + 1: raise ValueError( - f"{input.ndim=} != {(cell.spatial_ndim + 1)=}." - f"Expected input to {type(cell).__name__} to have `shape` (in_features, {'...'*cell.spatial_ndim})." - "Pass a single sample input to `tree_state`" + f"{input.ndim=} != {(cell.spatial_ndim+1)=}.\n" + f"Expected input to {type(cell).__name__} to have `shape` (in_features, {'... '*cell.spatial_ndim}).\n" + f"Pass a single sample input to `tree_state({type(cell).__name__}, input=...)`" ) if len(spatial_dim := input.shape[1:]) != cell.spatial_ndim: @@ -1816,33 +1503,3 @@ def _(cell: ConvGRUNDCell, *, input: Any) -> ConvGRUNDState: input = _check_rnn_cell_tree_state_input(cell, input) shape = (cell.hidden_features, *input.shape[1:]) return ConvGRUNDState(jnp.zeros(shape).astype(input.dtype)) - - -@tree_state.def_state(ScanRNN) -def _(rnn: ScanRNN, input: jax.Array | None = None) -> RNNState: - # the idea here is to combine the state of the forward and backward cells - # if backward cell exists. to have single state input for `ScanRNN` and - # single state output not to complicate the ``__call__`` signature on the - # user side. - input = [None] if input is None else input - # non-spatial cells don't need an input instead - # pass `None` to `tree_state` - # otherwise pass the a single time step input to the cells - return ( - tree_state(rnn.cell, input=input[0]) - if rnn.backward_cell is None - else concat_state(tree_state((rnn.cell, rnn.backward_cell), input=input[0])) - ) - - -@sk.tree_summary.def_type(ScanRNN) -def _(rnn: ScanRNN) -> str: - # display the type of the rnn cell and the type of the cell(s) it scans - # e.g. ScanRNN[SimpleRNNCell] instead of ScanRNN - return ( - f"{type(rnn).__name__}" - + "[" - + f"{type(rnn.cell).__name__}" - + (f",{type(rnn.backward_cell).__name__}" if rnn.backward_cell else "") - + "]" - ) diff --git a/serket/nn/__init__.py b/serket/nn/__init__.py index d4b9456..d2ccee8 100644 --- a/serket/nn/__init__.py +++ b/serket/nn/__init__.py @@ -145,10 +145,8 @@ FFTConvLSTM3DCell, GRUCell, LSTMCell, - RNNCell, - ScanRNN, SimpleRNNCell, - scan_rnn, + scan_cell, ) from serket._src.nn.reshape import ( CenterCrop1D, @@ -316,10 +314,8 @@ "FFTConvLSTM3DCell", "GRUCell", "LSTMCell", - "RNNCell", - "ScanRNN", "SimpleRNNCell", - "scan_rnn", + "scan_cell", # reshape "CenterCrop1D", "CenterCrop2D", diff --git a/tests/test_convolution.py b/tests/test_convolution.py index 94cabb2..bf5c1c4 100644 --- a/tests/test_convolution.py +++ b/tests/test_convolution.py @@ -1200,7 +1200,15 @@ def test_lazy_conv_local(): @pytest.mark.parametrize( - "sk_layer,keras_layer,kernel_size,strides,padding,dilation,ndim", + ( + "sk_layer", + "keras_layer", + "kernel_size", + "strides", + "padding", + "dilation", + "ndim", + ), [ *product( [sk.nn.Conv1D, sk.nn.FFTConv1D], diff --git a/tests/test_rnn.py b/tests/test_rnn.py index 54fedff..68d70fe 100644 --- a/tests/test_rnn.py +++ b/tests/test_rnn.py @@ -12,799 +12,208 @@ # See the License for the specific language governing permissions and # limitations under the License. -# testing against keras -# import tensorflow.keras as tfk -# import tensorflow as tf -# import numpy as np -# from serket._src.nn.recurrent import LSTMCell, ScanRNN +import os -# batch_size = 1 -# time_steps = 2 -# in_features = 3 -# hidden_features=2 +os.environ["KERAS_BACKEND"] = "jax" +from itertools import product -# inputs = np.ones([batch_size,time_steps, in_features]).astype(np.float32) -# inp = tf.keras.Input(shape=(time_steps, in_features)) -# rnn = (tf.keras.layers.LSTM(hidden_features, return_sequences=True, return_state=False))(inp) -# rnn = tf.keras.Model(inputs=inp, outputs=rnn) -# # rnn(inputs) -# w_in_to_hidden = jnp.array(rnn.weights[0].numpy()) -# w_hidden_to_hidden = jnp.array(rnn.weights[1].numpy()) -# b_hidden_to_hidden = jnp.array(rnn.weights[2].numpy()) -# x = jnp.ones([time_steps, in_features]) -# cell = LSTMCell(in_features, hidden_features, recurrent_weight_init="glorot_uniform", bias_init="zeros", -# weight_init="glorot_uniform") -# cell = cell.at["in_to_hidden.weight"].set(w_in_to_hidden) -# cell = cell.at["hidden_to_hidden.weight"].set(w_hidden_to_hidden) -# cell = cell.at["hidden_to_hidden.bias"].set(b_hidden_to_hidden) -# ScanRNN(cell, return_sequences=True)(x) ,rnn(inputs) - -# testing with keras -# inputs = np.ones([batch_size,time_steps, in_features]).astype(np.float32) -# inp = tf.keras.Input(shape=(time_steps, in_features)) -# rnn = tfk.layers.Bidirectional(tf.keras.layers.LSTM(hidden_features, return_sequences=False))(inp) -# rnn = tf.keras.Model(inputs=inp, outputs=rnn) -# # rnn(inputs) -# w_in_to_hidden = jnp.array(rnn.weights[0].numpy()) -# w_hidden_to_hidden = jnp.array(rnn.weights[1].numpy()) -# b_hidden_to_hidden = jnp.array(rnn.weights[2].numpy()) -# x = jnp.ones([time_steps, in_features]) -# cell = LSTMCell(in_features, hidden_features) -# cell = cell.at["in_to_hidden.weight"].set(w_in_to_hidden) -# cell = cell.at["hidden_to_hidden.weight"].set(w_hidden_to_hidden) -# cell = cell.at["hidden_to_hidden.bias"].set(b_hidden_to_hidden) - -# w_in_to_hidden_reverse = jnp.array(rnn.weights[3].numpy()) -# w_hidden_to_hidden_reverse = jnp.array(rnn.weights[4].numpy()) -# b_hidden_to_hidden_reverse = jnp.array(rnn.weights[5].numpy()) -# reverse_cell = LSTMCell(in_features, hidden_features) - -# reverse_cell = reverse_cell.at["in_to_hidden.weight"].set(w_in_to_hidden_reverse) -# reverse_cell = reverse_cell.at["hidden_to_hidden.weight"].set(w_hidden_to_hidden_reverse) -# reverse_cell = reverse_cell.at["hidden_to_hidden.bias"].set(b_hidden_to_hidden_reverse) - - -import jax import jax.numpy as jnp +import jax.random as jr +import keras import numpy.testing as npt import pytest -from serket._src.nn.recurrent import ( - ConvLSTM1DCell, - DenseCell, - FFTConvLSTM1DCell, - GRUCell, - LSTMCell, - ScanRNN, - SimpleRNNCell, -) - -# import pytest +import serket as sk -def test_vanilla_rnn(): +def test_simple_rnn(): + key = jr.PRNGKey(0) + time_step = 3 in_features = 2 hidden_features = 3 - # batch_size = 1 - time_steps = 10 - - # test against keras - # copy weights from keras to serket and compare outputs - # inputs = np.ones([batch_size,time_steps, in_features]).astype(np.float32) - # inp = tf.keras.Input(shape=(time_steps, in_features)) - # rnn = (tf.keras.layers.SimpleRNN(hidden_features, return_sequences=False, return_state=False))(inp) - # rnn = tf.keras.Model(inputs=inp, outputs=rnn) - - x = jnp.ones([time_steps, in_features]).astype(jnp.float32) - - w_in_to_hidden = jnp.array( - [[0.6252413, -0.34832734, 0.6286191], [0.84620893, 0.52448165, 0.13104844]] - ) - - w_hidden_to_hidden = jnp.array( - [ - [-0.24631214, -0.86077654, -0.44541454], - [-0.96763766, 0.24441445, 0.06276101], - [-0.05484254, -0.4464587, 0.893122], - ] - ) - - cell = SimpleRNNCell( - in_features=in_features, - hidden_features=hidden_features, - recurrent_weight_init="glorot_uniform", - key=jax.random.PRNGKey(0), - ) - - w_combined = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["in_hidden_to_hidden"]["weight"].set(w_combined.T) - sk_layer = ScanRNN(cell) - y = jnp.array([0.9637042, -0.8282256, 0.7314449]) - npt.assert_allclose(sk_layer(x), y) + input = jr.uniform(key, (time_step, in_features)) + keras_rnn = keras.layers.SimpleRNN( + hidden_features, + return_sequences=True, + return_state=True, + ) + serket_cell = sk.nn.SimpleRNNCell(in_features, hidden_features, key=key) + keras_output, *keras_state = keras_rnn(input[None]) + keras_output = keras_output[0] # drop batch dimension + i2h, h2h, b = keras_rnn.weights + serket_cell = ( + serket_cell.at["in_hidden_to_hidden"]["weight"] + .set(jnp.concatenate([i2h.numpy().T, h2h.numpy().T], axis=-1)) + .at["in_hidden_to_hidden"]["bias"] + .set(b.numpy()) + ) + serket_rnn = sk.nn.scan_cell(serket_cell) + serket_output, serket_state = serket_rnn(input, sk.tree_state(serket_cell)) + npt.assert_allclose(keras_output, serket_output, atol=1e-6) + npt.assert_allclose(keras_state[0][0], serket_state.hidden_state, atol=1e-6) def test_lstm(): - # tensorflow + key = jr.PRNGKey(0) + time_step = 3 in_features = 2 hidden_features = 3 - # batch_size = 1 - time_steps = 10 - - # inputs = np.ones([batch_size,time_steps, in_features]).astype(np.float32) - # inp = tf.keras.Input(shape=(time_steps, in_features)) - # rnn = (tf.keras.layers.LSTM(hidden_features, return_sequences=False, return_state=False))(inp) - # rnn = tf.keras.Model(inputs=inp, outputs=rnn) - - # w_in_to_hidden = jnp.array(rnn.weights[0].numpy()) - # w_hidden_to_hidden = jnp.array(rnn.weights[1].numpy()) - # b_hidden_to_hidden = jnp.array(rnn.weights[2].numpy()) - - x = jnp.ones([time_steps, in_features]).astype(jnp.float32) - - w_in_to_hidden = jnp.array( - [ - [ - -0.1619612, - -0.17861447, - -0.374527, - 0.21063584, - 0.1806348, - 0.0344786, - 0.44189203, - -0.55044144, - 0.28518462, - -0.09390897, - 0.56036115, - 0.19108337, - ], - [ - 0.03269911, - -0.21127799, - 0.55661833, - -0.6470987, - -0.27472985, - -0.21884575, - 0.2479667, - -0.34201348, - 0.00261247, - -0.6468279, - 0.5003185, - 0.6460693, - ], - ] - ) - - w_hidden_to_hidden = jnp.array( - [ - [ - 0.3196982, - 0.25284654, - -0.18152222, - 0.44958767, - -0.44068673, - -0.19395973, - -0.00905689, - -0.17610262, - 0.21773854, - -0.47118214, - -0.07700437, - 0.24598895, - ], - [ - -0.23678103, - -0.01854092, - -0.15681103, - -0.20309119, - -0.51169145, - 0.33006623, - 0.35155487, - 0.1802753, - -0.08975402, - -0.30867696, - 0.37548447, - -0.3264465, - ], - [ - -0.14270899, - 0.26242012, - -0.31327525, - 0.206014, - 0.5501963, - 0.14983827, - -0.15515868, - 0.2578809, - -0.14565073, - -0.33286166, - 0.4204296, - 0.21370588, - ], - ] - ) - - b_hidden_to_hidden = jnp.array( - [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - ) - - cell = LSTMCell( - in_features=in_features, - hidden_features=hidden_features, - recurrent_weight_init="glorot_uniform", - key=jax.random.PRNGKey(0), - ) - w_combined = jnp.concatenate([w_in_to_hidden.T, w_hidden_to_hidden.T], axis=1) - - cell = cell.at["in_hidden_to_hidden"]["weight"].set(w_combined) - cell = cell.at["in_hidden_to_hidden"]["bias"].set(b_hidden_to_hidden) - - sk_layer = ScanRNN(cell, return_sequences=False) - - y = jnp.array([0.18658024, -0.6338659, 0.3445018]) - npt.assert_allclose(y, sk_layer(x), atol=1e-5) - - w_in_to_hidden = jnp.array( - [ - [ - 0.11943924, - -0.609248, - -0.45503575, - -0.3439762, - -0.33675978, - 0.05291432, - -0.12904513, - -0.22977036, - 0.32492596, - 0.06835997, - 0.0484916, - 0.07520777, - ], - [ - 0.39872873, - -0.08020723, - -0.4879259, - -0.61926323, - -0.45951623, - -0.44556192, - -0.05298251, - 0.54848397, - 0.19754452, - 0.6012858, - -0.06859863, - 0.16502213, - ], - ] - ) - - w_hidden_to_hidden = jnp.array( - [ - [ - 0.18880641, - 0.21262297, - -0.2961502, - 0.33976135, - -0.09891935, - -0.00502901, - 0.34378093, - 0.4202192, - 0.36584634, - 0.08396737, - -0.4975226, - 0.15165171, - ], - [ - 0.30486387, - -0.46795598, - -0.07052832, - 0.51685417, - -0.23734125, - 0.1711132, - 0.16389124, - -0.08915165, - -0.02928232, - -0.2173849, - 0.19655496, - -0.45694238, - ], - [ - -0.1722902, - -0.23029403, - 0.05032581, - 0.21182823, - 0.5298174, - -0.50670344, - -0.18930247, - 0.30799994, - -0.18611868, - -0.08317372, - -0.26286182, - -0.30177474, - ], - ] - ) - - b_hidden_to_hidden = jnp.array( - [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - ) - - cell = LSTMCell( - in_features=in_features, - hidden_features=hidden_features, - recurrent_weight_init="glorot_uniform", - key=jax.random.PRNGKey(0), - ) - - w_combined = jnp.concatenate([w_in_to_hidden.T, w_hidden_to_hidden.T], axis=-1) - - cell = cell.at["in_hidden_to_hidden"]["weight"].set(w_combined) - cell = cell.at["in_hidden_to_hidden"]["bias"].set(b_hidden_to_hidden) - - sk_layer = ScanRNN(cell, return_sequences=True) - - y = jnp.array( - [ - [-0.07431775, 0.05081949, 0.07480226], - [-0.12263095, 0.07622699, 0.1146026], - [-0.15380122, 0.0886446, 0.13589925], - [-0.17376699, 0.0944333, 0.14736715], - [-0.18647897, 0.09689739, 0.1535385], - [-0.19453025, 0.09775667, 0.15683244], - [-0.19960524, 0.09789205, 0.1585632], - [-0.20278986, 0.0977404, 0.15945096], - [-0.20477988, 0.09750732, 0.15989034], - [-0.20601842, 0.09728104, 0.16009602], - ] - ) - - npt.assert_allclose(y, sk_layer(x), atol=1e-5) - - cell = LSTMCell( - in_features=in_features, - hidden_features=hidden_features, - recurrent_weight_init="glorot_uniform", - key=jax.random.PRNGKey(0), - ) - - sk_layer = ScanRNN(cell, return_sequences=True) - assert sk_layer(x).shape == (10, 3) - - -def test_gru(): - w1 = jnp.array( - [ - [ - -0.04667467, - 0.25340378, - 0.26873875, - 0.15961742, - 0.56519365, - 0.46263158, - -0.0030899, - 0.31380886, - 0.44481528, - ] - ] - ) - - w2 = jnp.array( - [ - [ - 0.23404205, - 0.10193896, - 0.27892762, - -0.488236, - -0.4173184, - -0.0588184, - 0.41350085, - 0.36151117, - -0.45407838, - ], - [ - -0.560196, - -0.22648495, - -0.12656957, - 0.31881046, - 0.47110367, - 0.30805635, - 0.41259462, - 0.40002275, - -0.0368616, - ], - [ - 0.5745573, - 0.4343021, - 0.42046744, - -0.09401041, - 0.5539224, - -0.13675115, - -0.5197817, - -0.21241805, - -0.16732433, - ], - ] - ) + input = jr.uniform(key, (time_step, in_features)) + keras_rnn = keras.layers.LSTM( + hidden_features, + return_sequences=True, + return_state=True, + ) + serket_cell = sk.nn.LSTMCell(in_features, hidden_features, key=key) + keras_output, *keras_state = keras_rnn(input[None]) + keras_output = keras_output[0] # drop batch dimension + i2h, h2h, b = keras_rnn.weights + + # serket combines the input and hidden weights + serket_cell = ( + serket_cell.at["in_hidden_to_hidden"]["weight"] + .set(jnp.concatenate([i2h.numpy().T, h2h.numpy().T], axis=-1)) + .at["in_hidden_to_hidden"]["bias"] + .set(b.numpy()) + ) + serket_rnn = sk.nn.scan_cell(serket_cell) + serket_output, serket_state = serket_rnn(input, sk.tree_state(serket_cell)) + npt.assert_allclose(keras_output, serket_output, atol=1e-6) + npt.assert_allclose(keras_state[0][0], serket_state.hidden_state, atol=1e-6) + npt.assert_allclose(keras_state[1][0], serket_state.cell_state, atol=1e-6) - cell = GRUCell(1, 3, bias_init=None, key=jax.random.PRNGKey(0)) - cell = cell.at["in_to_hidden"]["weight"].set(w1.T) - cell = cell.at["hidden_to_hidden"]["weight"].set(w2.T) - y = jnp.array([[-0.00142191, 0.11011646, 0.1613554]]) - ypred = ScanRNN(cell, return_sequences=True)(jnp.ones([1, 1])) - npt.assert_allclose(y, ypred, atol=1e-4) - - -@pytest.mark.parametrize("layer", [ConvLSTM1DCell, FFTConvLSTM1DCell]) -def test_conv_lstm1d(layer): - w_in_to_hidden = jnp.array( - [ - [ - [0.3159187, -0.37110862, 0.23497376], - [0.06916022, 0.16520068, -0.1498835], - ], - [ - [0.13892826, -0.2475906, 0.11548725], - [-0.14935534, 0.0077568, 0.31523505], - ], - [ - [0.20523027, 0.333159, -0.26372582], - [0.21769527, -0.28275424, 0.07145688], - ], - [ - [-0.32436138, 0.17985162, -0.05102682], - [-0.33781663, 0.07652837, 0.14034107], - ], - [ - [-0.2476197, 0.27073297, -0.15494357], - [-0.17142114, 0.0436784, -0.2635818], - ], - [ - [-0.1563589, -0.30193892, -0.3076105], - [0.30359367, -0.37472126, 0.08727607], - ], - [ - [0.02532503, -0.33569914, -0.16816947], - [-0.28197324, -0.20834318, -0.31490648], - ], - [ - [0.37559494, -0.10307714, -0.28350165], - [0.16282192, 0.25434867, 0.14521858], - ], - [ - [-0.3619054, -0.05932748, 0.13838741], - [0.317831, -0.01710135, 0.01839554], - ], - [ - [-0.33236656, -0.15234765, 0.23833898], - [-0.0525074, -0.1169591, 0.22625437], - ], - [ - [0.3350378, 0.3527101, -0.08017969], - [-0.25890553, 0.24611798, 0.30005935], - ], - [ - [-0.07834777, -0.02483597, -0.28757787], - [-0.15855587, 0.14020738, -0.3187018], - ], - ] - ) - - w_hidden_to_hidden = jnp.array( - [ - [ - [0.44095814, 0.12996325, 0.1313585], - [0.18582591, 0.07248487, -0.7859758], - [-0.17839126, 0.15680492, -0.08622836], - ], - [ - [-0.11601712, 0.00761805, 0.43996823], - [0.27362385, 0.0799137, 0.2580722], - [-0.563254, 0.19736156, 0.26167846], - ], - [ - [-0.28901652, -0.25223732, -0.10025343], - [0.56027263, -0.28712046, -0.18524358], - [0.37074035, 0.3996833, 0.1725195], - ], - [ - [0.07441625, 0.20128009, 0.30421543], - [-0.06981394, -0.17527759, 0.22605616], - [0.11372325, 0.63972735, -0.19949353], - ], - [ - [0.08129799, -0.06646754, -0.44094074], - [-0.09799376, 0.16513337, 0.1980969], - [-0.01823295, 0.33500522, 0.19564764], - ], - [ - [-0.4375121, -0.07695349, 0.27423194], - [0.25537497, 0.64107186, -0.09421141], - [0.21401826, -0.15687335, -0.07473418], - ], - [ - [-0.37147775, 0.06210529, -0.04531584], - [-0.38045418, 0.26204777, -0.17553791], - [-0.16380772, 0.39306286, -0.444068], - ], - [ - [-0.08250815, 0.5762788, 0.3014125], - [0.08091379, -0.20550683, 0.06467859], - [0.02479128, -0.16484486, 0.09149422], - ], - [ - [-0.1793791, 0.23342696, -0.33710676], - [0.4355502, -0.23507121, 0.11481185], - [-0.21538775, -0.16292992, -0.6203824], - ], - [ - [-0.1719443, 0.04258863, -0.35778967], - [0.12353352, 0.0826712, -0.10358769], - [-0.55321497, 0.07205058, 0.29797262], - ], - [ - [-0.52755165, 0.27079415, -0.04477403], - [-0.3376618, -0.32239383, -0.3393156], - [0.04485175, -0.04528336, 0.30485243], - ], - [ - [-0.14193594, -0.634814, 0.28351584], - [-0.16348608, -0.4000306, -0.08978741], - [-0.26926947, -0.12314601, -0.19621553], - ], - ] - ) - - b_in_to_hidden = jnp.array( - [ - [0.0], - [0.0], - [0.0], - [1.0], - [1.0], - [1.0], - [0.0], - [0.0], - [0.0], - [0.0], - [0.0], - [0.0], - ] - ) +def test_bilstm(): + key = jr.PRNGKey(0) + time_step = 3 in_features = 2 hidden_features = 3 - time_steps = 1 - spatial_dim = (3,) - - # inputs = np.ones([batch_size,time_steps, in_features,*spatial_dim]).astype(np.float32) - # inp = tf.keras.Input(shape=(time_steps, in_features,*spatial_dim)) - # rnn = (tf.keras.layers.ConvLSTM1D(hidden_features,recurrent_activation="sigmoid", kernel_size=3, padding='same', - # return_sequences=False,data_format='channels_first'))(inp) - # rnn = tf.keras.Model(inputs=inp, outputs=rnn) - - cell = layer( - in_features=in_features, - hidden_features=hidden_features, - recurrent_act="sigmoid", - kernel_size=3, + input = jr.uniform(key, (time_step, in_features)) + keras_rnn = keras.layers.LSTM( + hidden_features, + return_sequences=True, + return_state=True, + ) + + keras_rnn = keras.layers.Bidirectional( + keras.layers.LSTM( + hidden_features, + return_sequences=True, + return_state=False, + ) + ) + + keras_output = keras_rnn(input[None]) + + i2hf, h2hf, bf, i2hb, h2hb, bb = keras_rnn.weights + + i2hf = i2hf.numpy().T + h2hf = h2hf.numpy().T + ih2hf = jnp.concatenate([i2hf, h2hf], axis=-1) + bf = bf.numpy() + i2hb = i2hb.numpy().T + h2hb = h2hb.numpy().T + ih2hb = jnp.concatenate([i2hb, h2hb], axis=-1) + bb = bb.numpy() + + serket_cell = sk.nn.LSTMCell(in_features, hidden_features, key=key) + + forward_cell = ( + serket_cell.at["in_hidden_to_hidden"]["weight"] + .set(ih2hf) + .at["in_hidden_to_hidden"]["bias"] + .set(bf) + ) + backward_cell = ( + serket_cell.at["in_hidden_to_hidden"]["weight"] + .set(ih2hb) + .at["in_hidden_to_hidden"]["bias"] + .set(bb) + ) + + state1 = sk.tree_state(forward_cell) + output1, _ = sk.nn.scan_cell(forward_cell)(input, state1) + state2 = sk.tree_state(backward_cell) + output2, _ = sk.nn.scan_cell(backward_cell, reverse=True)(input, state2) + serket_output = jnp.concatenate([output1, output2], axis=1) + + npt.assert_allclose(keras_output[0], serket_output, atol=1e-6) + + +@pytest.mark.parametrize( + ("sk_layer", "keras_layer", "ndim"), + [ + *product( + [sk.nn.ConvLSTM1DCell, sk.nn.FFTConvLSTM1DCell], + [keras.layers.ConvLSTM1D], + [1], + ), + *product( + [sk.nn.ConvLSTM2DCell, sk.nn.FFTConvLSTM2DCell], + [keras.layers.ConvLSTM2D], + [2], + ), + *product( + [sk.nn.ConvLSTM3DCell, sk.nn.FFTConvLSTM3DCell], + [keras.layers.ConvLSTM3D], + [3], + ), + ], +) +def test_conv_lstm(sk_layer, keras_layer, ndim): + key = jr.PRNGKey(0) + time_step = 3 + in_features = 2 + spatial = [5] * ndim + kernel_size = 3 + hidden_features = 3 + input = jr.uniform(key, (time_step, in_features, *spatial)) + keras_rnn = keras_layer( + hidden_features, + kernel_size, + data_format="channels_first", padding="same", - weight_init="glorot_uniform", - recurrent_weight_init="glorot_uniform", - bias_init="zeros", - key=jax.random.PRNGKey(0), - ) - - cell = cell.at["in_to_hidden"]["weight"].set(w_in_to_hidden) - cell = cell.at["hidden_to_hidden"]["weight"].set(w_hidden_to_hidden) - cell = cell.at["in_to_hidden"]["bias"].set(b_in_to_hidden) - - x = jnp.ones([time_steps, in_features, *spatial_dim]) - - res_sk = ScanRNN(cell, return_sequences=False)(x) - - y = jnp.array( - [ - [-0.19088623, -0.20386685, -0.11864982], - [0.00493522, 0.18935747, 0.16954307], - [0.01413723, 0.00672858, -0.03464129], - ] - ) - - assert jnp.allclose(res_sk, y, atol=1e-5) - - cell = layer( - in_features=in_features, - hidden_features=hidden_features, - recurrent_act="sigmoid", - kernel_size=3, + use_bias=False, + return_sequences=True, + return_state=True, + ) + keras_output, *keras_state = keras_rnn(input[None]) + serket_cell = sk_layer( + in_features, + hidden_features, + kernel_size, padding="same", - weight_init="glorot_uniform", - recurrent_weight_init="glorot_uniform", - bias_init="zeros", - key=jax.random.PRNGKey(0), - ) - - res_sk = ScanRNN(cell, return_sequences=False)(x) - assert res_sk.shape == (3, 3) - - -def test_bilstm(): - # batch_size = 1 - time_steps = 2 - in_features = 3 - hidden_features = 2 - - x = jnp.ones([time_steps, in_features]) - cell = LSTMCell(in_features, hidden_features, key=jax.random.PRNGKey(0)) - reverse_cell = LSTMCell(in_features, hidden_features, key=jax.random.PRNGKey(0)) - - w_in_to_hidden = jnp.array( - [ - [ - -0.6061297, - 0.6038931, - 0.0219295, - -0.53232527, - 0.63680524, - -0.1877076, - 0.5494583, - 0.5319734, - ], - [ - -0.11174804, - 0.1967476, - -0.01281184, - 0.6291546, - -0.10848027, - -0.32045278, - 0.07772851, - -0.07741755, - ], - [ - 0.69948727, - -0.48679155, - 0.39291233, - -0.0054667, - 0.5324392, - 0.62987834, - -0.2530458, - -0.5623743, - ], - ] - ) - - w_hidden_to_hidden = jnp.array( - [ - [ - -0.07784259, - 0.5912869, - -0.08792564, - -0.07326522, - -0.07806911, - -0.75162244, - 0.01986005, - 0.24453232, - ], - [ - 0.23444527, - -0.5768899, - 0.24225983, - -0.23526284, - -0.2299888, - -0.444415, - 0.4977502, - 0.00633401, - ], - ] - ) - - b_hidden_to_hidden = jnp.array([0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0]) - - w_in_to_hidden_reverse = jnp.array( - [ - [ - 0.28273338, - -0.1472258, - 0.3937468, - 0.34040576, - -0.299861, - -0.38785607, - 0.00533426, - 0.06143087, - ], - [ - -0.40093276, - 0.39314228, - -0.43308863, - 0.532469, - -0.71949875, - 0.16529655, - -0.07926816, - -0.5383911, - ], - [ - -0.0023067, - -0.5820745, - 0.31508905, - 0.29104167, - -0.35113502, - -0.6884494, - 0.14833266, - -0.46562153, - ], - ] - ) - - w_hidden_to_hidden_reverse = jnp.array( - [ - [ - 3.12127233e-01, - 7.36315727e-01, - -1.91057637e-01, - 1.89247921e-01, - 4.54114564e-02, - 6.95739524e-04, - 5.34631252e-01, - 1.43038025e-02, - ], - [ - 3.68674904e-01, - -1.35606900e-01, - -3.05835426e-01, - -1.86572984e-01, - -7.80997992e-01, - 2.84251571e-02, - -1.41527206e-02, - 3.26157391e-01, - ], - ] - ) - - b_hidden_to_hidden_reverse = jnp.array([0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0]) - - combined_w = jnp.concatenate([w_in_to_hidden, w_hidden_to_hidden], axis=0) - cell = cell.at["in_hidden_to_hidden"]["weight"].set(combined_w.T) - cell = cell.at["in_hidden_to_hidden"]["bias"].set(b_hidden_to_hidden.T) - - combined_w_reverse = jnp.concatenate( - [w_in_to_hidden_reverse, w_hidden_to_hidden_reverse] - ) - reverse_cell = reverse_cell.at["in_hidden_to_hidden"]["weight"].set( - combined_w_reverse.T + key=key, + bias_init=None, + recurrent_act="sigmoid", ) - reverse_cell = reverse_cell.at["in_hidden_to_hidden"]["bias"].set( - b_hidden_to_hidden_reverse.T + w1, w2 = keras_rnn.weights + serket_cell = ( + serket_cell.at["in_to_hidden"]["weight"] + .set(jnp.transpose(w1.numpy(), (-1, -2, *range(ndim)))) + .at["hidden_to_hidden"]["weight"] + .set(jnp.transpose(w2.numpy(), (-1, -2, *range(ndim)))) ) - res = ScanRNN(cell, reverse_cell, return_sequences=False)(x) - - y = jnp.array([0.35901642, 0.00826644, -0.3015435, -0.13661332]) - - npt.assert_allclose(res, y, atol=1e-5) - - -def test_rnn_error(): - with pytest.raises(TypeError): - ScanRNN(None) - - with pytest.raises(TypeError): - ScanRNN(SimpleRNNCell(3, 3, key=jax.random.PRNGKey(0)), 1) - - layer = ScanRNN( - SimpleRNNCell(3, 3, key=jax.random.PRNGKey(0)), - SimpleRNNCell(3, 3, key=jax.random.PRNGKey(0)), - ) + state = sk.tree_state(serket_cell, input=input[0]) + sekret_output, serket_state = sk.nn.scan_cell(serket_cell)(input, state) - with pytest.raises(ValueError): - layer(jnp.ones([10, 3, 3])) + npt.assert_allclose(keras_output[0], sekret_output, atol=1e-6) + npt.assert_allclose(keras_state[0][0], serket_state.hidden_state, atol=1e-6) + npt.assert_allclose(keras_state[1][0], serket_state.cell_state, atol=1e-6) def test_dense_cell(): - cell = DenseCell( + cell = sk.nn.DenseCell( in_features=10, hidden_features=10, act=lambda x: x, weight_init="ones", bias_init=None, - key=jax.random.PRNGKey(0), + key=jr.PRNGKey(0), ) - x = jnp.ones([10, 10]) - res = ScanRNN(cell)(x) + input = jnp.ones([10, 10]) + state = sk.tree_state(cell) + output, _ = sk.nn.scan_cell(cell)(input, state) # 1x10 @ 10x10 => 1x10 - npt.assert_allclose(res, jnp.ones([10]) * 10.0) + npt.assert_allclose(output[-1], jnp.ones([10]) * 10.0)