src/WaveNet-for-language-processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = open('names.txt', 'r').read().splitlines()\n",
    "chars = ['.'] + sorted(list(set(\"\".join(words))))\n",
    "vocab_size = len(chars)\n",
    "\n",
    "stoi = {char:i for i, char in enumerate(chars)}\n",
    "itos = {i: char for char, i  in stoi.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "block_size = 8\n",
    "\n",
    "def build_dataset(words):\n",
    "    X, Y = [], []\n",
    "    for w in words:\n",
    "        context = [0]*block_size\n",
    "        for ch in w+'.':\n",
    "            ix = stoi[ch]\n",
    "            Y.append(ix)\n",
    "            X.append(context)\n",
    "            context = context[1:] + [ix]\n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    return X, Y\n",
    "\n",
    "import random\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any\n",
    "\n",
    "\n",
    "class Linear:\n",
    "  \n",
    "  def __init__(self, fan_in, fan_out, bias=True):\n",
    "    self.weight = torch.randn((fan_in, fan_out)) / fan_in**0.5\n",
    "    self.bias = torch.zeros(fan_out) if bias else None\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    self.out = x @ self.weight\n",
    "    if self.bias is not None:\n",
    "      self.out += self.bias\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "class BatchNorm1d:\n",
    "  \n",
    "  def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "    self.eps = eps\n",
    "    self.momentum = momentum\n",
    "    self.training = True\n",
    "    self.gamma = torch.ones(dim)\n",
    "    self.beta = torch.zeros(dim)\n",
    "    self.running_mean = torch.zeros(dim)\n",
    "    self.running_var = torch.ones(dim)\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    if self.training:\n",
    "      if x.ndim == 2:\n",
    "        dim = 0\n",
    "      elif x.ndim == 3:\n",
    "        dim = (0,1)\n",
    "      xmean = x.mean(dim, keepdim=True)\n",
    "      xvar = x.var(dim, keepdim=True) \n",
    "    else:\n",
    "      xmean = self.running_mean\n",
    "      xvar = self.running_var\n",
    "    xhat = (x - xmean) / torch.sqrt(xvar + self.eps) \n",
    "    self.out = self.gamma * xhat + self.beta\n",
    "    # update the buffers\n",
    "    if self.training:\n",
    "      with torch.no_grad():\n",
    "        self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n",
    "        self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.gamma, self.beta]\n",
    "\n",
    "\n",
    "class Tanh:\n",
    "  def __call__(self, x):\n",
    "    self.out = torch.tanh(x)\n",
    "    return self.out\n",
    "  def parameters(self):\n",
    "    return []\n",
    "  \n",
    "  \n",
    "class Embedding:\n",
    "  def __init__(self, num_embeddings, embbedding_dim):\n",
    "    self.weight = torch.randn((num_embeddings, embbedding_dim))\n",
    "    \n",
    "  def __call__(self, x):\n",
    "    self.out = self.weight[x]\n",
    "    return self.out\n",
    "  \n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.weight]\n",
    "  \n",
    "  \n",
    "class FlattenConsec:\n",
    "  \n",
    "  def __init__(self, n):\n",
    "    self.n = n\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    B, T, C = x.shape\n",
    "    x = x.view(B, T//self.n, C*self.n)\n",
    "    if x.shape[1]==1:\n",
    "      x = x.squeeze(1)  \n",
    "    self.out = x\n",
    "    return self.out\n",
    "    \n",
    "  def parameters(self):\n",
    "    return []\n",
    "\n",
    "\n",
    "class Sequential:\n",
    "  \n",
    "  def __init__(self, layers) -> None:\n",
    "    self.layers = layers\n",
    "    \n",
    "  def __call__(self, x) -> Any:\n",
    "    for layer in self.layers:\n",
    "      x = layer(x)\n",
    "    self.out = x\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [p for layer in self.layers for p in layer.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_embed = 10\n",
    "n_hidden = 200\n",
    "\n",
    "model = Sequential([Embedding(vocab_size ,n_embed),\n",
    "          FlattenConsec(2),\n",
    "          Linear(n_embed*2, n_hidden, bias=False),\n",
    "          BatchNorm1d(n_hidden), \n",
    "          Tanh(),\n",
    "          FlattenConsec(2),\n",
    "          Linear(n_hidden*2, n_hidden, bias=False),\n",
    "          BatchNorm1d(n_hidden), \n",
    "          Tanh(), \n",
    "          FlattenConsec(2),\n",
    "          Linear(n_hidden*2, n_hidden, bias=False),\n",
    "          BatchNorm1d(n_hidden), \n",
    "          Tanh(), \n",
    "          Linear(n_hidden, vocab_size)\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    model.layers[-1].weight *= 0.1\n",
    "\n",
    "\n",
    "parameters = model.parameters()\n",
    "for p in parameters:\n",
    "    p.requires_grad = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0 step: 3.2829\n",
      "  10000 step: 2.4425\n",
      "  20000 step: 2.3414\n",
      "  30000 step: 1.9970\n",
      "  40000 step: 1.8514\n",
      "  50000 step: 2.1139\n",
      "  60000 step: 1.8516\n",
      "  70000 step: 2.1510\n",
      "  80000 step: 1.9696\n",
      "  90000 step: 2.0510\n",
      " 100000 step: 2.0503\n",
      " 110000 step: 1.9292\n",
      " 120000 step: 1.8501\n",
      " 130000 step: 1.8797\n",
      " 140000 step: 2.0601\n",
      " 150000 step: 1.8939\n",
      " 160000 step: 1.7829\n",
      " 170000 step: 1.5893\n",
      " 180000 step: 1.6906\n",
      " 190000 step: 2.0437\n"
     ]
    }
   ],
   "source": [
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "  \n",
    "  ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "  Xb, Yb = Xtr[ix], Ytr[ix] \n",
    "  \n",
    "  logits = model(Xb)\n",
    "  loss = F.cross_entropy(logits, Yb)\n",
    "  \n",
    "  # backward pass\n",
    "  for p in parameters:\n",
    "    p.grad = None\n",
    "  loss.backward()\n",
    "  \n",
    "  lr = 0.1 if i < 150000 else 0.01\n",
    "  for p in parameters:\n",
    "    p.data += -lr * p.grad\n",
    "\n",
    "  if i % 10000 == 0: \n",
    "    print(f'{i:7d} step: {loss.item():.4f}')\n",
    "  lossi.append(loss.log10().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x6fffea8d5e80>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhlUlEQVR4nO3dd3zU9eHH8dddxmUvQhISAmFvCBCIICJqAC1WRK1IrSC1Wrc2liq1gq21QWstVim0WkfBQf0V3KIShgJhJYRN2ISVBWSTed/fH5ccXAlkkOSS8H4+Hnk0+d7n+83nwxXu7WeaDMMwEBEREWnBzM6ugIiIiEhtFFhERESkxVNgERERkRZPgUVERERaPAUWERERafEUWERERKTFU2ARERGRFk+BRURERFo8V2dXoDFYrVZOnDiBr68vJpPJ2dURERGROjAMg4KCAsLDwzGbL92H0iYCy4kTJ4iMjHR2NURERKQBjh49SseOHS9Zpk0EFl9fX8DWYD8/PyfXRkREROoiPz+fyMhI++f4pbSJwFI9DOTn56fAIiIi0srUZTqHJt2KiIhIi6fAIiIiIi2eAouIiIi0eAosIiIi0uIpsIiIiEiLp8AiIiIiLZ4Ci4iIiLR4CiwiIiLS4imwiIiISIunwCIiIiItngKLiIiItHgKLCIiItLitYnDD5tKaUUlf16WRmmFledu7ou7q/KdiIiIM+gTuBZvrTnEwvVHOFte6eyqiIiIXLEUWC7B3cVM9YnXpRUKLCIiIs6iwHIJJpMJD1cXAErLrU6ujYiIyJVLgaUWFjfbH1GJhoREREScRoGlFtU9LCXqYREREXEaBZZaeFT1sGgOi4iIiPMosNTCw009LCIiIs6mwFILi6vmsIiIiDibAkstLNU9LBoSEhERcRoFllpUDwlpWbOIiIjzKLDUwqN6SEg9LCIiIk6jwFILTboVERFxPgWWWmjSrYiIiPMpsNTi3BwWBRYRERFnUWCpxbmN4zQkJCIi4iwKLLU4N4dFPSwiIiLOosBSi3NzWNTDIiIi4iwKLLXw0MZxIiIiTqfAUguLNo4TERFxugYFlnnz5hEVFYWHhwexsbFs3LjxomWXLFlCTEwMAQEBeHt7Ex0dzcKFCx3KFBYW8uijj9KxY0c8PT3p27cvCxYsaEjVGp02jhMREXE+1/resHjxYuLj41mwYAGxsbHMnTuX8ePHk5aWRkhIyAXlg4KCePbZZ+nduzfu7u588cUXTJ8+nZCQEMaPHw9AfHw8K1asYNGiRURFRfHtt9/y8MMPEx4ezi233HL5rbwMFk26FRERcbp697C8+uqr3H///UyfPt3eE+Ll5cXbb79dY/kxY8YwadIk+vTpQ7du3XjiiScYOHAga9assZdZt24d06ZNY8yYMURFRfHAAw8waNCgS/bcNBcPTboVERFxunoFlrKyMpKTk4mLizv3ALOZuLg4kpKSar3fMAwSExNJS0tj9OjR9usjR47ks88+4/jx4xiGwcqVK9m7dy/jxo2r8TmlpaXk5+c7fDUV+8Zx2odFRETEaeo1JJSTk0NlZSWhoaEO10NDQ9mzZ89F78vLyyMiIoLS0lJcXFz4+9//ztixY+2vv/766zzwwAN07NgRV1dXzGYzb775pkOoOV9CQgK///3v61P1BtNOtyIiIs5X7zksDeHr60tqaiqFhYUkJiYSHx9P165dGTNmDGALLOvXr+ezzz6jc+fOfP/99zzyyCOEh4c79OZUmzlzJvHx8faf8/PziYyMbJK66ywhERER56tXYAkODsbFxYXMzEyH65mZmYSFhV30PrPZTPfu3QGIjo5m9+7dJCQkMGbMGM6ePctvf/tbli5dyoQJEwAYOHAgqampvPLKKzUGFovFgsViqU/VG+zcPiwaEhIREXGWes1hcXd3Z+jQoSQmJtqvWa1WEhMTGTFiRJ2fY7VaKS0tBaC8vJzy8nLMZsequLi4YLU6PyRUnyWkHhYRERHnqfeQUHx8PNOmTSMmJobhw4czd+5cioqKmD59OgBTp04lIiKChIQEwDbfJCYmhm7dulFaWspXX33FwoULmT9/PgB+fn5ce+21zJgxA09PTzp37szq1av597//zauvvtqITW0YTboVERFxvnoHlsmTJ5Odnc2sWbPIyMggOjqaZcuW2SfipqenO/SWFBUV8fDDD3Ps2DE8PT3p3bs3ixYtYvLkyfYyH330ETNnzuTuu+/m9OnTdO7cmRdffJEHH3ywEZp4earnsFRaDcorrbi5aHNgERGR5mYyDMNwdiUuV35+Pv7+/uTl5eHn59eozy4pr6T3c8sA2P78OHw93Br1+SIiIleq+nx+q7ugFtU9LKDN40RERJxFgaUWJpPJHlpKdZ6QiIiIUyiw1IF9abN6WERERJxCgaUOtHmciIiIcymw1MG5pc0KLCIiIs6gwFIH1ZvHlWpISERExCkUWOrg3Pb86mERERFxBgWWOjg3h0U9LCIiIs6gwFIH51YJqYdFRETEGRRY6sDiqmXNIiIizqTAUgf2SbeawyIiIuIUCix1oB4WERER51JgqYPqHhbNYREREXEOBZY60LJmERER51JgqQNtHCciIuJcCix1UD2HRZNuRUREnEOBpQ7OzWFRD4uIiIgzKLDUgTaOExERcS4FljrwsA8JqYdFRETEGRRY6sCiZc0iIiJOpcBSB+c2jlNgERERcQYFljrQpFsRERHnUmCpA20cJyIi4lwKLHVQHVi0cZyIiIhzKLDUgcVVpzWLiIg4kwJLHZzbh0U9LCIiIs6gwFIHOq1ZRETEuRRY6qB647gKq0FFpXpZREREmpsCSx1UbxwH2u1WRETEGRRY6qC6hwU0LCQiIuIMCix1YDabcHepmseiHhYREZFmp8BSR9XDQqXqYREREWl2Cix1dO48IfWwiIiINDcFljqyL23W5nEiIiLNrkGBZd68eURFReHh4UFsbCwbN268aNklS5YQExNDQEAA3t7eREdHs3DhwgvK7d69m1tuuQV/f3+8vb0ZNmwY6enpDalekzi3eZwCi4iISHOrd2BZvHgx8fHxzJ49m5SUFAYNGsT48ePJysqqsXxQUBDPPvssSUlJbNu2jenTpzN9+nS++eYbe5kDBw4watQoevfuzapVq9i2bRvPPfccHh4eDW9ZI/Owz2HRkJCIiEhzMxmGYdTnhtjYWIYNG8Ybb7wBgNVqJTIykscee4xnnnmmTs8YMmQIEyZM4IUXXgDgrrvuws3Nrcael7rIz8/H39+fvLw8/Pz8GvSM2vxkwTo2HT7DvJ8OYcLADk3yO0RERK4k9fn8rlcPS1lZGcnJycTFxZ17gNlMXFwcSUlJtd5vGAaJiYmkpaUxevRowBZ4vvzyS3r27Mn48eMJCQkhNjaWTz755KLPKS0tJT8/3+GrqXXw9wTg2JniJv9dIiIi4qhegSUnJ4fKykpCQ0MdroeGhpKRkXHR+/Ly8vDx8cHd3Z0JEybw+uuvM3bsWACysrIoLCxkzpw53HjjjXz77bdMmjSJ2267jdWrV9f4vISEBPz9/e1fkZGR9WlGg0S18wLg8CkFFhERkebm2hy/xNfXl9TUVAoLC0lMTCQ+Pp6uXbsyZswYrFbbnJCJEyfyq1/9CoDo6GjWrVvHggULuPbaay943syZM4mPj7f/nJ+f3+ShpXM7bwCOnCpq0t8jIiIiF6pXYAkODsbFxYXMzEyH65mZmYSFhV30PrPZTPfu3QFbGNm9ezcJCQmMGTOG4OBgXF1d6du3r8M9ffr0Yc2aNTU+z2KxYLFY6lP1yxYVbAssh3MUWERERJpbvYaE3N3dGTp0KImJifZrVquVxMRERowYUefnWK1WSktL7c8cNmwYaWlpDmX27t1L586d61O9JlU9JHQir0RLm0VERJpZvYeE4uPjmTZtGjExMQwfPpy5c+dSVFTE9OnTAZg6dSoREREkJCQAtvkmMTExdOvWjdLSUr766isWLlzI/Pnz7c+cMWMGkydPZvTo0Vx33XUsW7aMzz//nFWrVjVOKxtBkLc7vh6uFJRUkH66mJ6hvs6ukoiIyBWj3oFl8uTJZGdnM2vWLDIyMoiOjmbZsmX2ibjp6emYzec6boqKinj44Yc5duwYnp6e9O7dm0WLFjF58mR7mUmTJrFgwQISEhJ4/PHH6dWrF//9738ZNWpUIzSxcZhMJqLaebP9eB6Hc4oUWERERJpRvfdhaYmaYx8WgEc/SOGLbSd59kd9uH901yb7PSIiIleCJtuH5UrXpWri7SGtFBIREWlWCiz1oKXNIiIizqHAUg9dgqs2j8vR5nEiIiLNSYGlHqp7WE7kndXSZhERkWakwFIP7bzd8bG4Yhg6U0hERKQ5KbDUg8lkIqpqWOiQhoVERESajQJLPWnirYiISPNTYKmnc6c2K7CIiIg0FwWWeuoYWHWmUG6Jk2siIiJy5VBgqacO/h4AnMg96+SaiIiIXDkUWOopIsATUGARERFpTgos9dShKrDkl1RQWFrh5NqIiIhcGRRY6snH4oqfh+2Q65PqZREREWkWCiwNEF49LJSnibciIiLNQYGlAcI1j0VERKRZKbA0QPVKIQ0JiYiINA8Flgao7mE5rr1YREREmoUCSwOEB1T1sOSph0VERKQ5KLA0QLi/5rCIiIg0JwWWBjh/lZBhGE6ujYiISNunwNIAoX4emExQVmHlVFGZs6sjIiLS5imwNIC7q5n2PhYATmrirYiISJNTYGmgcyuFNI9FRESkqSmwNJBWComIiDQfBZYGql4pdFLb84uIiDQ5BZYG6qAhIRERkWajwNJAEQHanl9ERKS5KLA0UFSwNwC7TuZTVFrh5NqIiIi0bQosDdQr1Jeodl6UlFtZvjvT2dURERFp0xRYGshkMnFLdAQAn6aecHJtRERE2jYFlstwy6BwAL7fm81p7XgrIiLSZBRYLkP3EB/6hftRYTX4avtJZ1dHRESkzVJguUwTo229LJ9t1bCQiIhIU1FguUw3D7QFlo2HTrP1aK5zKyMiItJGNSiwzJs3j6ioKDw8PIiNjWXjxo0XLbtkyRJiYmIICAjA29ub6OhoFi5ceNHyDz74ICaTiblz5zakas0uPMDT3ssS/59USsornVwjERGRtqfegWXx4sXEx8cze/ZsUlJSGDRoEOPHjycrK6vG8kFBQTz77LMkJSWxbds2pk+fzvTp0/nmm28uKLt06VLWr19PeHh4/VviRM//uB/tfS0cyC7ilW/SnF0dERGRNqfegeXVV1/l/vvvZ/r06fTt25cFCxbg5eXF22+/XWP5MWPGMGnSJPr06UO3bt144oknGDhwIGvWrHEod/z4cR577DHef/993NzcGtYaJwn0duel2wcA8K+1h9h2LNe5FRIREWlj6hVYysrKSE5OJi4u7twDzGbi4uJISkqq9X7DMEhMTCQtLY3Ro0fbr1utVu655x5mzJhBv3796lOlFuP63qHc2C8Mw4Dlu2vubRIREZGGca1P4ZycHCorKwkNDXW4Hhoayp49ey56X15eHhEREZSWluLi4sLf//53xo4da3/9pZdewtXVlccff7xO9SgtLaW0tNT+c35+fn2a0WRGdm/Hsp0ZmnwrIiLSyOoVWBrK19eX1NRUCgsLSUxMJD4+nq5duzJmzBiSk5N57bXXSElJwWQy1el5CQkJ/P73v2/iWtffoI4BAGw7lothGHVuj4iIiFxavYaEgoODcXFxITPT8eyczMxMwsLCLv5LzGa6d+9OdHQ0Tz31FHfccQcJCQkA/PDDD2RlZdGpUydcXV1xdXXlyJEjPPXUU0RFRdX4vJkzZ5KXl2f/Onr0aH2a0WR6d/DFzcXEmeJyjp3RKc4iIiKNpV6Bxd3dnaFDh5KYmGi/ZrVaSUxMZMSIEXV+jtVqtQ/p3HPPPWzbto3U1FT7V3h4ODNmzKhxJRGAxWLBz8/P4aslsLi60KeDrS5bNfFWRESk0dR7SCg+Pp5p06YRExPD8OHDmTt3LkVFRUyfPh2AqVOnEhERYe9BSUhIICYmhm7dulFaWspXX33FwoULmT9/PgDt2rWjXbt2Dr/Dzc2NsLAwevXqdbnta3aDOgaw7VgeW4/m2jeVExERkctT78AyefJksrOzmTVrFhkZGURHR7Ns2TL7RNz09HTM5nMdN0VFRTz88MMcO3YMT09PevfuzaJFi5g8eXLjtaIFGdjRH4Ctx/KcXBMREZG2w2QYhuHsSlyu/Px8/P39ycvLc/rw0N7MAsb99Xu83F3Y/vx4XMyaeCsiIlKT+nx+6yyhRtatvQ/e7i4Ul1WyP6vQ2dURERFpExRYGpmL2UT/iOphoVznVkZERKSNUGBpAoMiAwBI1QZyIiIijUKBpQnEdA4EYN3+HCfXREREpG1QYGkCI7sH42o2cfhUMYdzipxdHRERkVZPgaUJ+FhciYmy9bKs3pvt5NqIiIi0fgosTWRMrxBAgUVERKQxKLA0kWt7tgdg3YEcSsornVwbERGR1k2BpYn0DvMl1M9CSbmVTYdPO7s6IiIirZoCSxMxmUz2XpZVaRoWEhERuRwKLE3o2p62eSxfbz/JmaIyJ9dGRESk9VJgaULX9mpPmJ8HJ/JKmPbORgpKyp1dJRERkVZJgaUJ+VhcWXjfcAK93Nh2LI/73ttMWYXV2dUSERFpdRRYmliPUF8W3heLr8WVjYdO8681h5xdJRERkVZHgaUZ9I/w5/lb+gHwt8R9HM89S0l5JV9sO0FWfomTayciItLyuTq7AleK24ZEsHjTUTYePk384lSyC0s5mF3E+H6h/OOeGGdXT0REpEVTD0szMZlM/OHWfriYTWw4dJqD2bYzhrYfy3NyzURERFo+BZZm1DvMj8ev74HJBJMGRwBwIq+EfK0eEhERuSQNCTWzJ+J68OCYrlhcXVh/8BQn80rYl1nA0M5Bzq6aiIhIi6UeFiewuLoA0DPUF4C0jEJnVkdERKTFU2Bxol5htsCyN7PAyTURERFp2RRYnOhcD4sCi4iIyKUosDhRr1D1sIiIiNSFAosTdQ/xwWSCU0Vl5BSWOrs6IiIiLZYCixN5urvQOcgLgL0aFhIREbkoBRYnq57HskeBRURE5KIUWJxMK4VERERqp8DiZPaVQgosIiIiF6XA4mTVPSw7T+Sz+fBpJ9dGRESkZVJgcbLu7X0Y0bUdZRVWfvavDXy57SRHTxeTV6zzhURERKqZDMMwnF2Jy5Wfn4+/vz95eXn4+fk5uzr1draskoffT2ZlWrb9mskEb0wZwoSBHZxYMxERkaZTn89v9bC0AJ7uLvxzagxTR3Qm0MsNdxczhgHf7MxwdtVERERaBAWWFsLNxcwfJvZny6xxvDktBoAdx/OcXCsREZGWQYGlBRoQ4Q/AwZwi8ks0l0VERESBpQUK8nYnIsATgJ3H851cGxEREedrUGCZN28eUVFReHh4EBsby8aNGy9adsmSJcTExBAQEIC3tzfR0dEsXLjQ/np5eTlPP/00AwYMwNvbm/DwcKZOncqJEycaUrU2o7qXRcNCIiIiDQgsixcvJj4+ntmzZ5OSksKgQYMYP348WVlZNZYPCgri2WefJSkpiW3btjF9+nSmT5/ON998A0BxcTEpKSk899xzpKSksGTJEtLS0rjlllsur2Wt3ICOtsCyTYFFRESk/suaY2NjGTZsGG+88QYAVquVyMhIHnvsMZ555pk6PWPIkCFMmDCBF154ocbXN23axPDhwzly5AidOnWq9XmtfVlzTb7fm83UtzfSJdiblb8e4+zqiIiINLomW9ZcVlZGcnIycXFx5x5gNhMXF0dSUlKt9xuGQWJiImlpaYwePfqi5fLy8jCZTAQEBNT4emlpKfn5+Q5fbU31kNAhTbwVERGpX2DJycmhsrKS0NBQh+uhoaFkZFx8z5C8vDx8fHxwd3dnwoQJvP7664wdO7bGsiUlJTz99NNMmTLlomkrISEBf39/+1dkZGR9mtEqBHq70zHQNvFW81hERORK1yyrhHx9fUlNTWXTpk28+OKLxMfHs2rVqgvKlZeXc+edd2IYBvPnz7/o82bOnEleXp796+jRo01Ye+fRxFsREREb1/oUDg4OxsXFhczMTIfrmZmZhIWFXfQ+s9lM9+7dAYiOjmb37t0kJCQwZswYe5nqsHLkyBFWrFhxybEsi8WCxWKpT9VbpQEd/fl6RwaL1qeTmV/KhIEdGNIp0NnVEhERaXb16mFxd3dn6NChJCYm2q9ZrVYSExMZMWJEnZ9jtVopLS21/1wdVvbt28fy5ctp165dfarVZo3sFgxA+uli/rXmED9ZkMS+zAIn10pERKT51auHBSA+Pp5p06YRExPD8OHDmTt3LkVFRUyfPh2AqVOnEhERQUJCAmCbbxITE0O3bt0oLS3lq6++YuHChfYhn/Lycu644w5SUlL44osvqKystM+HCQoKwt3dvbHa2upERwbw1ePXkHo0l4+Tj7IlPZeXlqXxVtXW/SIiIleKegeWyZMnk52dzaxZs8jIyCA6Opply5bZJ+Kmp6djNp/ruCkqKuLhhx/m2LFjeHp60rt3bxYtWsTkyZMBOH78OJ999hlgGy4638qVKx2Gja5EfcP96Bvux/AuQYyf+z3Ld2ey4eApYruqF0pERK4c9d6HpSVqi/uw1OS3S7fzwYZ0BkUGsPShkZjNJofXV6ZlcbqwjNuGRGAymS7yFBERkZahyfZhEed6Mq4HXu4ubD2ay02v/cCnqcexWm15M6ewlAf+vZmnPt7K0i3HnVxTERGRxqXA0oqE+Hrwp0kD8LW4kpZZwBMfpfLX5XsB+L/kY5RX2sLLc5/s4HBOkTOrKiIi0qgUWFqZWwdHsOaZ63l4TDcA/vn9QTLySvhoYzoAAV5uFJVV8sRHWyivtDqzqiIiIo1GgaUV8vd0Y8b4XgztHEhphZUHFm7m8KlifCyu/N+DI/H3dGPrsTw+Tb2yT7wWEZG2Q4GllTKZTMwY3wuAbcdsO+HeOjic7iE+TBsZBdgm4YqIiLQFCiyt2FVd23FNj2D7z1OG2062HtOrPQA/7M2mQsNCIiLSBiiwtHJP39gbi6uZq7u3o1+47eyhQR0D8Pd0I7+kgq3Hcp1bQRERkUagwNLK9Y/wZ83T1/OvacPs11zMJnvPy+q0bGdVTUREpNEosLQB7X0teLi5OFwb0ysEgFV7FVhERKT1U2Bpo0ZX9bBsO5bHwexC3ll7iA0HTzm5ViIiIg1T77OEpHUI8fOgbwc/dp3MZ+xfv6fSauDn4Uryc2Nxc1FOFRGR1kWfXG1Y9Wqhyqrt+/NLKkg+csaZVRIREWkQ9bC0Yb8c3Y3iskoGdwpgxZ4sPk09waq0bK7SSc8iItLKqIelDfP3cuP5W/oxMTqC63tXTcI9bzO5NnBQt4iIXCEUWK4Q1/Roj8kEezIKyMgr4Y0V++j13DKW7TgJQHmllT9+sYu31xxyck1FREQupCGhK0SQtzuDOgaQejSX1xL3sXhTOlYDfv3xNnqF+fHP7w/w4cajmExw+5CO+Hu5ObvKIiIiduphuYJcV7U3y4cbbWHF3dVMYWkFt89fx4cbjwJgGLDp8GlnVlNEROQCCixXkOpVQwDBPhY+e/Rq2nm7c7qoDIBwfw8A1mu/FhERaWEUWK4gAyL86VAVSl6c1J/eYX78bcpgQnwt/HJ0V56+qTcAGw6ph0VERFoWzWG5gpjNJhbeN5ysglJGdrPthHt192A2/PYGTCYTmfklAOw8kUfe2XL8PTWPRUREWgb1sFxhuof42sNKNZPJBEConwddgr2xGrBZ81hERKQFUWARB7FdggDNYxERkZZFgUUcVO+Cq3ksIiLSkiiwiIPYrrYelh3H80j4ajfvrD1EUWmFk2slIiJXOk26FQcd/D3p1t6bA9lF/OP7gwAczC7ihVv713rvij2ZdGvvQ+d23k1dTRERucKoh0Uu8I97hjJjfC+mDO8EwOJNRzmZd9ahTEZeCct3ZdrPI0pJP8PP393M9Hc36YwiERFpdAoscoHuIb48cl13Em4bQGyXIMoqrcxfdcD++pmiMm77+1p+8e/NrDtgm5y7qWrOy8HsIlKP5jqj2iIi0oYpsMglPRHXA4CPNh4lI68EwzCY8X9bOZFn27MlqSqwbD+eZ7/n09QTzV9RERFp0xRY5JJGdG3H8Kpelmlvb+Th91NYvjvL/vqWo2cA2Hki337ti20nqKi0NntdRUSk7VJgkUsymUz8ZnwvLK5m0jIL+HpHBgA/u8o2v2Xr0Tzyiss5lFMEgK+HKzmFZfahIhERkcagVUJSq5ioIJbHX0vSgVOkpJ8hMsiLX47uypKU4xSWVvBJ6nEAIgI8ub53CAvXH+HT1BOM7tm+lieLiIjUjXpYpE4ig7y4c1gkc24fyCPXdcfVxcygjgEALFx/BID+EX5MjA4HYNmOkyQf0eZzIiLSOBRYpMEGdwoAYH9WIWA7DXpIp0AGdvSnqKySO/+xngWrD2C1apmziIhcHgUWabDBnQIdfu4f4Y/ZbOKD+6/ilkHhVFoN5ny9h5+/t4lThaVk5JXw1g8HWb0320k1FhGR1qpBgWXevHlERUXh4eFBbGwsGzduvGjZJUuWEBMTQ0BAAN7e3kRHR7Nw4UKHMoZhMGvWLDp06ICnpydxcXHs27evIVWTZhQdGeDwc/8IfwB8LK68dlc0CbcNwOJqZlVaNtf/ZTVXv7SCP365m2lvb+TV7/aq50VEROqs3oFl8eLFxMfHM3v2bFJSUhg0aBDjx48nKyurxvJBQUE8++yzJCUlsW3bNqZPn8706dP55ptv7GVefvll/va3v7FgwQI2bNiAt7c348ePp6SkpOEtkybX3tdCx0BPADr4exDsY7G/ZjKZmDK8E588cjVd23uTd7acSqtBr1BfAP6WuI/HPtpyQWgxDINF64/w3a7M5muIiIi0eCajnvuox8bGMmzYMN544w0ArFYrkZGRPPbYYzzzzDN1esaQIUOYMGECL7zwAoZhEB4ezlNPPcWvf/1rAPLy8ggNDeXdd9/lrrvuqvV5+fn5+Pv7k5eXh5+fX32aI5fpsQ+38PnWE4ztG8qbU2NqLFNUWsHXOzKIjgyge4gPH28+yrNLd1BWaeWD+2MZ2S3YXvbr7Sd56P0U3F3NbHluLN4WLWQTEWmr6vP5Xa8elrKyMpKTk4mLizv3ALOZuLg4kpKSar3fMAwSExNJS0tj9OjRABw6dIiMjAyHZ/r7+xMbG3vRZ5aWlpKfn+/wJc5xZ0xHfD1cuWNox4uW8bbYXu8e4gPAT2Ii+fEg22qilXvO9cyVlFfy4le7ASirsLJ2f04T1lxERFqTegWWnJwcKisrCQ0NdbgeGhpKRkbGRe/Ly8vDx8cHd3d3JkyYwOuvv87YsWMB7PfV55kJCQn4+/vbvyIjI+vTDGlE1/Roz/bnxzO+X1i97ruhTwgAiecFlrd+OMixM+cOWVyZpsm5IiJi0yyrhHx9fUlNTWXTpk28+OKLxMfHs2rVqgY/b+bMmeTl5dm/jh492niVlWZxTY9gXM0mDmYXcSiniIy8Ev5edcDibUMiAFiVlqWTn0VEBKjnTrfBwcG4uLiQmek4ITIzM5OwsIv/F7bZbKZ79+4AREdHs3v3bhISEhgzZoz9vszMTDp06ODwzOjo6BqfZ7FYsFgsNb4mrYOvhxuxXYNYu/8UK/ZkkXzkNMVllQztHMifJg3gq+0nOZlXwp6MAvp00LwkEZErXb16WNzd3Rk6dCiJiYn2a1arlcTEREaMGFHn51itVkpLSwHo0qULYWFhDs/Mz89nw4YN9XqmtD7X97YNAy5YfYCvtmfgYjbxwsT+eLi5cHXVRNyVaTWvPhMRkStLvYeE4uPjefPNN3nvvffYvXs3Dz30EEVFRUyfPh2AqVOnMnPmTHv5hIQEvvvuOw4ePMju3bv5y1/+wsKFC/nZz34G2Ja/Pvnkk/zxj3/ks88+Y/v27UydOpXw8HBuvfXWxmmltEg39LbNY8kusIXXX4zqQt9wW2/KmKrXVu1xnMeyJyOfgpLyZqyliIi0BPVeMzp58mSys7OZNWsWGRkZREdHs2zZMvuk2fT0dMzmczmoqKiIhx9+mGPHjuHp6Unv3r1ZtGgRkydPtpf5zW9+Q1FREQ888AC5ubmMGjWKZcuW4eHh0QhNlJYqKtibru29OZhdRESAJ0/E9bC/Nqbq4MTk9DNkF5TS3tfCBxvS+e3S7cT1CeWtaTUvoT5fQYlt75cAL/cma4OIiDSPeu/D0hJpH5bWa9H6I/z1u728PmUwI7sHO7w24W8/sPNEPoM7BfDbH/Xh7jc3UFZpxWyCDb+No72vhTe/P8jXO07yz6kxDhvXVVRaueHV1eSfLeerJ66hg79njb+/qLSC8kqrQo2IiBM02T4sIo3tZ1d1Jvm5sReEFYC5k6Px93RjS3oud/4jibJKKwBWw3Ya9KnCUv78bRop6bn8Z7PjSrHNR85w5FQxZ4rL+dNXewDYfiyP6e9sZMPBU/ZyP31zPWNeWcWpwtImbKWIiFwuBRZpsXqE+vL2vcPwcDNjGNAx0JNHr7OtNvt820k+3JhOWYUtxHy57aTDvcvP29r/860neG/dYX72rw2sTMtmwWrb8ukTuWfZeiyP3OJy1h883UytEhGRhlBgkRZtaOdA3p42jB8NCOPNqTH8NLYTAJsOn+adtYft5XaeyOdQThFg21H5u922wFK9u+7sz3aSd9Y2WXfzkTNYrQbJR87Y7z//exERaXkUWKTFG9k9mL/fPZQ+HfwID/BkaOdADANOFZUR7GPhqq5BAHy13dbLsj+rkCOninF3MfPez4cT6OUGQO8wX7zdXSgoqSAts+B/Aot6WEREWjIFFml1fjzw3AaDd8d2YtJg2864X1QNC1X3rozs3o6IAE/emjaMX4zqwsL7YhnSORCAzYdPk5J+LrDsPJHP2bLK5mqCiIjUkwKLtDo/GtABdxczFlczd1/ViXF9w3A1m9h9Mp/9WYX2+StxfWxL7Yd2DuR3N/elva+FYVG23pjVe3PYecJ2aKa3uwsVVoOtx3Kd0h4REaldvfdhEXG2ED8P/vPgCFxMJkJ8bXv1XN09mNV7sxk/93sqrbaV+tUHLJ4vJsrWw5K4JxPDgDA/D4Z0DuCr7RkkHznDVV3bNV9DRESkztTDIq1SdGQAAzr6239+YHRXgrzd7WFleFRQjXuvDI4MxNVsonr3oaGdAxna2dbrkqKJtyIiLZZ6WKRNuLp7MMm/iyMzv5TDp4roHeZbYzlPdxf6R/iTejQXgCGdAxlaNa8lOd22eshsNtXpdxqGwfzVB/D3dOPu2M6N0g4REamZelikzTCZTIT5e3BV13aX3Ll2WNWwENh6WPqF++HhZia3uJyDVUujz3cgu7DGVUTvrjvMy8vS+N0nO3S+kYhIE1NgkStO9cRbi6uZvh38cHMxM7BjAOC4AV1ucRm/+2Q7Y19dzR0LkhyWQe88kUdC1Q66hgEHsi8MOiIi0ngUWOSKM7pne27qH8avxvbE3dX2V+CuYZEAzFu5nz0Z+Ww7lkvcq6tZtD4dq2ELJe9vOAJAcVkFj324xX5UAMDezILmb4iIyBVEgUWuOB5uLsz/2VAevLab/dqkwRHE9QmhrNLKQ4tSmPLP9eQUltE9xIffTegD2Damyztbzhsr9nMwu4gwPw8mRocDts3qRESk6SiwiGCb/5Jw20CCvN05lFNEUVklI7u145NHrua+UV3oGepDSbmVeSv389YPhwD4w8R+DO9iG17apx4WEZEmpcAiUqW9r4WXbx+Ip5sLtwwK553pw/CxuGIymbhrmO0Mo39+f5CySitXd2/H2L6h9AixrUba9z89LJsOn2bsq6t59du0Zm+HiEhbpMAicp64vqFsnT2Ov00ZjMXVxX79tiER9vkuZhPMurkfJpOJHlWHKx47c5ai0goAPt58lJ++uZ59WYXMX32AU4WlF/ye6lOmRUSkbhRYRP5HdTA5X4CXOzcPsJ1hdHdsZ3pV7fMS6O1OsI9tCfWB7EKWpBxjxv9to7zSwN3FTHmlwf8lH7M/xzAM/r5qP31nLeOdtYeaoTUiIm2DAotIHT0/sR+v3RXNczf3dbjevaqXZV9mIf9aYwsh946M4vcT+wHw4cZ0rFYDwzCY8/UeXl6WRoXV4O+rDqinRUSkjhRYROrIz8ONidERF/TAVM9j+WLbCXaeyMfNxcQTN/TglkHh+FhcOXyqmOW7M3lycSr/+P4gAF7uLmQXlPL1jpNUVFr5xXub+PHraygpt50YXVhawe8/38nuk/nN20gRkRZKgUXkMvUMtfWwrEzLBuD63iEEervjbXHl1sG2Zc+/XJTMp6knMJvgpdsH8MvRtiXV7607zLyVB1i+O4vtx/PYkp4LwPvrj/DO2sP86avdzd8gEZEWSIFF5DJ1D3E8t+j2IR3t3/90uO2MIcOAcH8PFv9yBJOHdWJKbCRuLiZS0nN5LXGvvfzOE3kAbDtm+9+UI2eoqNSwkYiIAovIZepR1cMCEOTtzpheIfaf+4b78fSNvfn51V34+onR9mMBQnw9+FHVJF6rAd7uthVJ249XBZbjuQAUlVWyJ8O2x0tZhZWzZZVN3h4RkZZIgUXkMrXzdifQyw2AWwaFXzDH5aEx3Zj14774V5WpNv3qLphMEOpn4cVJAwDYcTyPM0VlHD191l5u82HbwYv3vbeJ2D8tJyOvpCmbIyLSIrk6uwIirZ3JZCKuTyhf78jgZ1d1qvN90ZEBfPrI1YT6eWA2mQA4mFPE+oOnHMptPnKGq7q144d9OQAs353Jz67q3HgNEBFpBdTDItIIXr5jIJt/F3fBfJbaDOwYQKifB+19LYT6WTAMWLz5KAAd/D0A2Hz4DEtTjtvvWVMVXGpy5FQR3+7MaEALRERaNgUWkUZgMpnwcHOpveAlDIjwB2D1XttqoynDO+FqNpGRX8IHG9Pt5dYeyKGi0kpWQQn3vbuJT1NtYSa/pJw7/5HEAwuT+W5X5mXVRUSkpVFgEWkh+oXbAoth2H4e3iWIfuF+ABSUVODv6YafhysFJRVsO57H31ceIHFPFk/9ZyubD5/mz8vSyMy3HQOgXXRFpK1RYBFpIap7WABMJugX7sfQzkH2axMGduDq7sEAfLH1JIs32YaOKqwGDyxMZtGGI/Z71x04pROkRaRNUWARaSH6nxdYugZ74+vhRkxUoP3apMERXNOjPQDvrDvE2fJKeoX60q29N6eLyjAMuGNoR8b1DQXg30lHmrcBIiJNSIFFpIUI9bPYD1Ic2DEAgBFd2xHk7c6gjv7EdA7kmh62HpbqYaMHx3Rlwc+G4mtxJcTXwrM/6sO0kVEA/DflGPkl5Y1ez+pzkUREmpOWNYu0ECaTiejIAJbvzmJwpwDAdhr0D7+5DhezCZPJRGSQF1HtvDh8qpgwPw9uHhiOm4uZlTPG4Go2EeDlzoiu7egR4sO+rEKu+lMiXYK9mTCwA/eN6oLF9fImBgM8uCiZLUdzWf6ray/YW2bt/hw83V0Y0inwIneLiDSMelhEWpDfTejLb27sxeRhkfZr3hZXhxVI1TvkPjSmG24utr/CwT4WArxsvTMmk4lfje2Ju6uZ4rJKdp7I5+VlafzotR/4blcmpRW23XINw2BfZgEvLdvD9a+s4nefbK+15ySroIRvd2WSXVBKytEzDq8dzz3L1Lc3MuWf6zldVHb5fxgiIudRD4tICxIV7M3DY7pfssyTcT25eWA4fTpcfM+XHw3oQFyfUI6eKWbTodO88m0aB7KLuP/fm/Fyd6FHiA+HcorIL6mw33Mwp4hBHQP4SUzkRZ97/h4wh3OKoNe5177fm02l1aDSarAk5Ri/uKZrHVosIlI36mERaWXcXc30DffDVLU77qXKdWvvw13DO5H41Bh+MaoLIb4Wissq2Xosj/ySClzNJuL6hDC5KqQ8/9lOjpwquugzv6/aIwbgUI5jufPDzAcb0zXPRUQaVYMCy7x584iKisLDw4PY2Fg2btx40bJvvvkm11xzDYGBgQQGBhIXF3dB+cLCQh599FE6duyIp6cnffv2ZcGCBQ2pmojUwN/Tjd/d3JcNv72BLx4bxbyfDuHrJ65h5x/G89a0YfzptgEM7xJEUVklj36whd0n8ymtqORfaw7xo9d+4POtJ7BaDdbsPxdKzg8slee9ZjLBwewiNhw6XWu9TheV8Y/VB5j+zka2Hctt9HaLSNtR7yGhxYsXEx8fz4IFC4iNjWXu3LmMHz+etLQ0QkJCLii/atUqpkyZwsiRI/Hw8OCll15i3Lhx7Ny5k4iICADi4+NZsWIFixYtIioqim+//ZaHH36Y8PBwbrnllstvpYgAtvkt/SP8HZZQA7iYTbx65yBueu0Hth/P46bXfiDI290+F+W3S7cT4OVGTuG5uSkHs88Flu3H88g7W46fhys39g/jP5uP8cGGdK7q2q7GehiGwd8S9zNv5X7KKq326+9MH+5Qbv3BU3y8+Rizbr7w8EgRubLUu4fl1Vdf5f7772f69On2nhAvLy/efvvtGsu///77PPzww0RHR9O7d2/eeustrFYriYmJ9jLr1q1j2rRpjBkzhqioKB544AEGDRp0yZ4bEWlcHQO9WPrwSCYM7IDJZOv9CPG10CnIi4KSCp74KBWwHdoIcCLvLCXltgm8P1QNFY3sFsw9V0UBsGxHBqlHcy/4PYZhkPD1Hv66fC9llVZ6hdrm4qw9cIrisgqHsnO+3sN/U47ZN8UTkStXvQJLWVkZycnJxMXFnXuA2UxcXBxJSUl1ekZxcTHl5eUEBZ3bwXPkyJF89tlnHD9+HMMwWLlyJXv37mXcuHE1PqO0tJT8/HyHLxG5fN1DfJn30yGseGoMcydHs2rGGObcPgDA3tsyaXAEvh6uGAakny4GsJ8kfU3PYAZ09GdYVCBllVZu+/ta/vjFLnYcz6OkvJLkI2d46uOt/PP7gwC8MLEfy568ho6BnpRVWB3mwZwtq2TH8Tzg0gc+isiVoV6BJScnh8rKSkJDQx2uh4aGkpFRtxNin376acLDwx1Cz+uvv07fvn3p2LEj7u7u3HjjjcybN4/Ro0fX+IyEhAT8/f3tX5GRF1/VICL11yXYm1sHR+Dl7srIbsGM7Xvu7/zonu3pGuwN2IaFCksrSEm3LXEeXbUT7z/uieHW6HCsBry15hA3v76G3s8t4/b561hSdfL0nyYN4J4RUZhMJuL62J6fuDvL/ntSj+ZSYbVN3N185PQFvS8icmVp1lVCc+bM4aOPPmLp0qV4eHjYr7/++uusX7+ezz77jOTkZP7yl7/wyCOPsHz58hqfM3PmTPLy8uxfR48eba4miFyRZt7UG1+LKwM7+hPVzouoqsByKKeIdftzqLAaRLXzIjLIC4Agb3fm3jWYt++NYWS3dvh72uaf+FpcuWVQOAvvG85PYzvZn39DH9v8t8Q9WVirQkrykXOTdssrjTpN4hWRtqtek26Dg4NxcXEhM9Px6PrMzEzCwsIuee8rr7zCnDlzWL58OQMHDrRfP3v2LL/97W9ZunQpEyZMAGDgwIGkpqbyyiuvOPTEVLNYLFgslvpUXUQuQ9f2Pqz+zXV4urlgMpnoYg8shew+aRuSvb536AX3Xd87lOt7h2IYBjmFZfh7uuHueuF/J8V2aYePxZWcwlK2HstlcKdANh229dp4ubtQXFbJmn05XNfrwon9InJlqFcPi7u7O0OHDnWYMFs9gXbEiBEXve/ll1/mhRdeYNmyZcTExDi8Vl5eTnl5OWazY1VcXFywWq2ISMsQ5O2Op7ttx93qwLLjeD7f7bL9B8zE6PCL3msymWjva6kxrIBtz5jRPW3nJCXuzqLSapByxBZYqs9GWrMvh1OFpUx9eyMvLdvTKG0Skdaj3kNC8fHxvPnmm7z33nvs3r2bhx56iKKiIqZPnw7A1KlTmTlzpr38Sy+9xHPPPcfbb79NVFQUGRkZZGRkUFhYCICfnx/XXnstM2bMYNWqVRw6dIh3332Xf//730yaNKmRmikijalrsA8Au07mc7a8kqh2Xgzs6F/LXZd2Q1UPzSepx0k9mktBaQU+FlfuG9UFkwnSMgu4+60NfL83m/mrDnC0asKviFwZ6h1YJk+ezCuvvMKsWbOIjo4mNTWVZcuW2Sfipqenc/LkSXv5+fPnU1ZWxh133EGHDh3sX6+88oq9zEcffcSwYcO4++676du3L3PmzOHFF1/kwQcfbIQmikhjiwr2cvh5YnRErTvv1mZ8/zA6+Htw7MxZHv0gBYDBnQII9rHQP9wWhvZkFNjLf5x87KLPyioooaAJTqoWEecxGW1g/+z8/Hz8/f3Jy8vDz8/P2dURuSLE/HE5OYWlACQ+dS3d2vtc9jPX7s/h7rc22H+OH9uTx2/owUvL9jB/1QE83Vz42VWdePOHQ4T7e/DD09eTfOQMb685hJ+nK94WVzYeOs3OE/lEBHjy1ePXaMM5kRasPp/fOvxQRBqka7A3OYWlDIjwb5SwAnB192B+MaoLb605BEBM50AApo+MIjOvhDuHRRIdGcB/Nh/jRF4J76w9xGvL91FQeuGS5+O5Z3n+8538dXJ0o9Ttcmw7lsuejAJ+MrTjZfdEiVypFFhEpEGGdA5k4+HTTB7WuPsg/Xp8L/tW/0OqAkuInwevnhc8Jg2O4N11h/njl7ttdekUwPW9QzhTXE6fDn6083bnvvc2sXTLcW7sH8b4fpdexXgxi9Yf4WTeWR6/oQcWV5cGPaOkvJJ739nE6aIywvw8GN2zfYOeI3KlU2ARkQZ5Mq4H4/uF2rfqbywebi589MBVl+yJuDMmknfXHQagczsv3po2jCBvd4cyv7y2G/NXHeDZpdsZ3CmAEF8PDMMg+cgZOgV5EeLnUcOTz8kuKOW5T3dgGLD1aB7/uGco3hbbP5mGYfDZ1hN0budda/v/m3LMvkvwNzszFFhEGqhZN44TkbbDw82FwZ0Cm2SIo7Zn9g33I65PKGF+HvyrhrACtkDVO8yXnMIyHlyYTGlFJQlf7+GOBUlc8/JK/vTVbs4UldXwdJvvdmVSPcNvzf4cfvrmek7knrWfhfTER6nc9+4mKiovvv2C1Wrwr6rhrepnVm+MJyL1o0m3ItJqWa0GZvPFw82hnCImvrGG/JIKugZ7czCnyOH1QC83Xpw0gB8N6IDVapBdWEqIrwWTycS0tzeyem82twwK54d92ZwpLsfX4sqY3iF8vvWE/RmLH7iK2IucSr1iTyY/f3czvhZXDKCwtIKlD49kcKfARmk/2Obq+Hq44uehycXS+tTn81s9LCLSal0qrIBtg7s3fjoEswl7WHlhYj/euXcYvUJ9OVNczsPvp3Db39cy5I/fEfunRP6WuJ/8knLWHbAduPj4DT1Y8vDVDO4UQEFphT2sRAR4Atg3zvtfVqthP+RxSmwnru3V/oLymw6f5qbXfuCnb6536HmpPgW7NtuO5XLdn1cx7W2dbC9tnwKLiLRpo3u254Vb+xPia7EfuHhd7xA+f2wUj17XHbMJUtJzyS227dvy+op9/HP1QcorDbq196Z7iA9dgr35vwdHMvOm3oT5efDMTb15dkIfAL7bncn/dlRvPnyaifPWsv7gaVzMJqaNjGJc1QGS3+7KJK+4nNmf7uDOfySx+2Q+6w6cYneG7YiDT1OP02fWMl5bvu+S7bJaDWZ/tpOySitb0nMvObwl0hZo0q2ItHl3x3bmp8M7OcyNcXc18+vxvbixfxibD59mQMcA5q86wPLdmbyxcj+Aw+oiF7OJX17bjV9e2w2wDe+4u5g5cqqYA9mFdA/x5WTeWRK+2sNnVb0wvhZXZv24LxEBnvj2DsHNxcT+rEJGvbyCghLbUuwALzdyi8v5YV8O/cL9+XBjOoYBf12+ly7tvbllUM1HHizdcpwt6bn2n1OP5XJdrxCSj5zhgw3p/PLarvQM9W3UP0cRZ1IPi4hcES42kbd/hD/3Xt2FoZ0D+cPEfni5n1u+fKnl0D4WV0Z0s81d+XZXJu+tO8z1r6zms60nMJngrmGRrPj1GH4SY1v27efhxlVVc10KSiroGerDovtieeKGHoDtrKTc4jL7oY8AMz7eyvZjeRf87oKScuZUnafkW7VyKbUqvLy0bA//TTnGxDfWsnTLxXcDFmlt1MMiIlIlPMCTp8b14oUvdhER4Fnr+UhxfUNZvTebucv3UVZhWy0U0zmQ2T/ux4Aa7v31uF6YTCZ+1D+MO4Z2xNXFTJi/bXn1xsOn+XpHBpVWg56hPkQEeLIyLZtf/SeVb58c7TBf54MN6WQXlNIl2Ju7Yzvxxy93k3o0l/yScpKrDo08W17JrxZvJSOvlIfGdGusPyIRp1EPi4jIee4dGUXCbQOY/7MhtS6vjusTAkBZhRVXs4nnbu7Lxw+OqDGsAAyKDODfPx/OXcM74epi++e3W3tvwv09KKuwMnf5XgDG9Q1j7l2D8bW4sj+rkFV7sxyes2xnBgA/H9WF4V2CAEg9msuafTlUWg26Bnvz2PXdAdvQkg6KlLZAgUVE5DwuZhNThndiYMeAWst28PfktsERdG3vzeJfXlV1snT99qUxmUxc08O2gigz33Y2U1zfUPw93bhruG046c3vz+3lklVQQurRXADG9gmld5gf7q5m8s6W8+7awwCM6RVC/NiejOjajrIKKy9VDR+JtGYKLCIil+HVydGseGoMQzsHNfgZo3oE279v72thYISth+beq7vgYjaRdPAUO47b5rKs2J2FYcDAjv6E+Xvg7mqmf7ht/4qNh08DMKZXe0wmE7+7uQ8mE3yx7STJR043uH4iLYECi4iIk13dPZjqjpm4PiH2+SoRAZ5MGNABgLd+sO3pUr2Py9g+ofb7z9+IztPNxT5M1C/cn8lVk35/83/b2J9V0LQNEWlCCiwiIk4W5O3OsChbyPhRVUCpdv81XQH4fNtJPtlynDX7bRvaje13LrCcf57RiG7t8HA7t9IpflxP2nm7cyC7iB/9bQ3//P7ABfvG1EV+STmvfreXZ/67jSUpx8gqKKmx3ILVB/jdJ9sveWSBSENolZCISAvwxpTBHMgusi+Vrjagoz93DO3I/yUf48nFqQBEBnnS67w9Vs4PLGN6OR6uGOLrwZePX8MzS7axKi2bP321h5zCMmbe1PuS822OnCrib4n7CfJ2I8jbwttrD5FdYJtj89Gmo7iaTXz84AiH3p3DOUXM+do2X+aGPqFc1yukQX8WIjVRYBERaQFC/DwueoL0nNsGUF5p5dNU24Z0Y/uEOYSNjoGedA/x4diZYm44b6ioWpi/B+/cO4x31x3m95/v4p/fH8TiaubhMd2xuJrZl1VISvoZotp52wPTnK/38PWODIfndAn25obeISzfncnhU8UsWp/uEFg+2Jhu//6LrScVWKRRKbCIiLRwri5m/vKTQXi4uvDVjpPcOayjw+smk4kP7o+luLTSfsbR/zKZTEy/ugsm4PnPd/H6iv28vmI/ZhNUH2Pkajax4qkxeLq72OfKTBkeSXZBKUM7B/HzUVFYXF24aUAYt89P4usdJ3nh1n54ubtSUl7Jx5uP2n/ft7syKK3oj8XVpYbaiNSfAouISCvg6mLmpTsGMuf2ATUO5YT4ekAdduK/9+ouWA145ds0issqsRq2ibreFldyCkt5Y+U+ugT7UGE1GNwpgITbBl7wjCGdAolq58XhU8Us25HBbUM68tX2k5wpLiciwJNKq0FGfgnf781hbN8Le3xEGkKBRUSkFanvPi81+fmoLky/OoqiskoKSsoJ9rGw7Vget89fx39TjhPs4w7AlOGdLlqH24Z05NXv9rIk5Ti3DenIovVHqu6J5HRROW+vPcQX204osEij0SohEZErkMlkwsfiSgd/T9xczAztHMjonu2ptBpk5pfia3Hl5oEdLnr/pMERAKw9kMMv3ttMSnourmYTdw6L5OZBtvu+25XJ2bLKZmmPtH0KLCIiAsCTcT3s308cHI6X+8U74SODvIjtEoRhwPLdmZhN8NS4XoT4ejA4MoCIAE+Kyyr545e7KClXaJHLp8AiIiKAbW7KLYPC8XJ34d6RUbWW/+W1XXExmxjVPZivnxhtP2TRZDLZv39/Qzq3zlur84zkspmMhuwg1MLk5+fj7+9PXl4efn5+zq6OiEirVWk1qLBa67y6p6LSaj/I8X8l7s7kN/+3jVNFZUwaHMFfJ0c3Yk2lLajP57d6WERExM7FbKrXUuSLhRWwbR734qT+ABzKKbrsusmVTYFFRESaTAd/274wGXk1b+UvUlcKLCIi0mTC/G2792YXllJpbfUzEMSJFFhERKTJBPtYcDGbqLQa5BSWOrs60oopsIiISJNxMZsI8bUAcFLDQnIZFFhERKRJhVYd6qh5LHI5FFhERKRJdaiax5KZr8AiDafAIiIiTaq6h0VDQnI5FFhERKRJhamHRRqBAouIiDSpMM1hkUbQoMAyb948oqKi8PDwIDY2lo0bN1607Jtvvsk111xDYGAggYGBxMXF1Vh+9+7d3HLLLfj7++Pt7c2wYcNIT09vSPVERKQFqe5hyVAPi1yGegeWxYsXEx8fz+zZs0lJSWHQoEGMHz+erKysGsuvWrWKKVOmsHLlSpKSkoiMjGTcuHEcP37cXubAgQOMGjWK3r17s2rVKrZt28Zzzz2Hh4dHw1smIiItwvk9LG3g+DpxknoffhgbG8uwYcN44403ALBarURGRvLYY4/xzDPP1Hp/ZWUlgYGBvPHGG0ydOhWAu+66Czc3NxYuXNiAJujwQxGRlqykvJLezy0DYOvscfh7ujm5RtJSNNnhh2VlZSQnJxMXF3fuAWYzcXFxJCUl1ekZxcXFlJeXExQUBNgCz5dffknPnj0ZP348ISEhxMbG8sknn1z0GaWlpeTn5zt8iYhIy+Th5kKAly2kaOKtNFS9AktOTg6VlZWEhoY6XA8NDSUjI6NOz3j66acJDw+3h56srCwKCwuZM2cON954I99++y2TJk3itttuY/Xq1TU+IyEhAX9/f/tXZGRkfZohIiLNLExLm+UyNesqoTlz5vDRRx+xdOlS+/wUq9UKwMSJE/nVr35FdHQ0zzzzDDfffDMLFiyo8TkzZ84kLy/P/nX06NFma4OIiNRf9V4smQos0kCu9SkcHByMi4sLmZmZDtczMzMJCwu75L2vvPIKc+bMYfny5QwcONDhma6urvTt29ehfJ8+fVizZk2Nz7JYLFgslvpUXUREnMg+8VZDQtJA9ephcXd3Z+jQoSQmJtqvWa1WEhMTGTFixEXve/nll3nhhRdYtmwZMTExFzxz2LBhpKWlOVzfu3cvnTt3rk/1RESkhape2qwhIWmoevWwAMTHxzNt2jRiYmIYPnw4c+fOpaioiOnTpwMwdepUIiIiSEhIAOCll15i1qxZfPDBB0RFRdnnuvj4+ODj4wPAjBkzmDx5MqNHj+a6665j2bJlfP7556xataqRmikiIs6k3W7lctU7sEyePJns7GxmzZpFRkYG0dHRLFu2zD4RNz09HbP5XMfN/PnzKSsr44477nB4zuzZs3n++ecBmDRpEgsWLCAhIYHHH3+cXr168d///pdRo0ZdRtNERKSl0G63crnqvQ9LS6R9WEREWrbdJ/O56bUf8Pd0I3XWWEwmk7OrJC1Ak+3DIiIi0hBd23vj6eZC3tlydp8scHZ1pBVSYBERkSZncXVhZLd2AKzaW/NRLiKXosAiIiLNYkyv9gCsTst2ck2kNVJgERGRZnFtzxAAko+coaCk3Mm1kdZGgUVERJpFp3ZedA32psJqsHb/KWdXR1oZBRYREWk2o3tWDQvt1bCQ1I8Ci4iINJtz81iyaAO7akgzUmAREZFmc1XXdlhczZzIK+GjTTq4VupOgUVERJqNh5sL06/uAsDMJdt5e80hJ9dIWgsFFhERaVZP39iL+6+xhZY/fLGLP3y+i7IKq5NrJS2dAouIiDQrk8nEb3/Uh1/F9QTg7bWH+Mk/kjh6urhO9//l2zT+8PkurNba58B8tDGda/+8kgPZhZdVZ3E+BRYREWl2JpOJJ+J68ObUGPw93dh6NJebXvuB/2w6esnJuGkZBby+Yj9vrz3E/6Ucu+TvMAyDvyXu48ipYj5NPdHYTZBmpsAiIiJOM7ZvKF8+PophUYEUllbwm/9u4zf/t+2i5T9NPW7//qWv95B39uIb0G09lseJqtOhtx/LbbQ6i3MosIiIiFN1DPTiowdGMPOm3phM8HHyMU7mnb2gnNVq2HtKPNzMnCoqY+7yvRd97tfbT9q/3348T8uoWzkFFhERcToXs4lfXtuN/uH+AGw8dPqCMinpZzieexYfiyt/u2swAP9OOlLj/BTDMPh6R4b955zCMk5W9bZI66TAIiIiLcbwLkFAzYHlk6rhoPH9whjXL4zRPdtTaTX4ctvJC8ruOplP+uliLK5mugZ7A7DtWF4T1rx12JtZwN9X7aekvNLZVak3BRYREWkxLhZYyiut9mAyMTocgBv7hQE1b/P/9XZb78qYXu3tz9xx/MoOLBl5Jfz0zfW8vCyNP3+T5uzq1JsCi4iItBjDomzhYl9WIacKSwHb3JXZn+3kTHE5wT4WRnZrB8C1Vdv8b0k/Q17xucm3hmHw1Q5buLmpfwf6R9iGmbZVBZa6LIduiB3H87hx7ves2ZdTa1mr1eCf3x/gu12ZTVKX/1VaUclD7yeTU1gGwHvrDrMvs6BZfndjUWAREZEWI8jbnZ6hPgBsOnyG8kor8f9J5YMN6ZhM8OyE3ri62D66IgI86RHig9WAH/af62VJSc/lYHYRHm5mru8TwsCOtsCy/Vgux3PPct1fVnHnP5IcQs7FlFZUkl1QWqe6f7gxnT0ZBfzzh4O1ll2+O5M/fbWHXy1OpbKRA5RhGGw7luuwGd+LX+5mS3oufh6uDO8SRIXV4PnPd7aqicgKLCIi0qJUD+FsOHSKZ5du55PUE7iaTbx212AmDe7oUPba6tOf084Flg82pANw88Bw/Dzc6BXmi5uLiTPF5dz37iaOnCpm46HTTHtnI4WlFZesyyPvp3D1nBWs2197r8muk/kAbD58utade99ZexiAwtIK0jIu7OkoKa9scE/QvJX7ueWNtbxZFZxKyit5v+rPZO5d0bxyxyDcXc2s3X+Kb3ZmXOpRLYoCi4iItCjDu9iGfD7cmM5/Nh/DbIK/3z2EWwaFX1B2TK8QwDaPxTAM8orL+WKbbenzlOGdALC4utA7zA+APRkFeLq5EODlRurRXG6c+z3XvbKK6/+yiuX/MzxztqySVWnZlFVaeerjrZfc86XSarDnpC14FJdVsu1/9n1564eDxP5pOT/sy2b3yXySDp6yv7bl6BmHstuO5dJ/9je83IB5JoWlFfzze1tQ2VA1D+hgdhGVVgN/Tzeu6xVCp3Ze/LzqPKf/phy/6LNaGgUWERFpUYZXzWMpKbf1Uvz2R30YVzXB9n/FRAXi6eZCVkEpu08W8EnqcUorrPQK9WVIpwB7uQFVw0IAz07ow6L7YvH1cOXYmbMcyiniYHYRD72f7DCnZMvRM1RU9XKczCvhuU92sPVoLh9uTOdQTpFDPQ7lFHH2vJU3SQfOBZKS8kreWLmfzPxSHlyYzB+/3AXYlnIDbEnPdXjWF9tOUmE1+HBjOhWV9Ttj6cMN6eSX2HqN9lb13OzLsv1vjxAfTCbb7xxTNf9n14n8ej3fmRRYRESkRQnz96BL1VLkO4Z25L5RXS5a1sPNhRFVk3BnLtlmHwaZMjzS/uEMcE33YABu6B3C3bGd6B/hz/L4a/n73UNY/MBV3DywA+WVBg+/n2xfdbTpkK3no08HP1zMJj7beoKJ89Yyc8l2Jv8jidNFZfbn7zzhuAJp3XmB5ZudGeRWzZcpKqtk7X7baw9e2xWw7S9zvuoVUnlny0k+4vjapZRWVPLWmnPzZzLyS8grLmdv1eTaHqG+9tf6VPU4Hc89e8meo5ZEgUVERFqcV34ykKdv7M2Lk/o7BI+aVA8VbT2Wx7EzZ7G4mi+Y63Jj/zC+eGwUC+4Zan9eqJ8HPxrQgdiu7Zg7OdoeWn5fNRl102FbcPjp8Ejix9oOavT3dKOdtztZBaXM+HirfdJq9fyV6hVMyeln7HudfLTxKAD3jepCr6rQMCDCn1+MsgWWg9lF5Bbbwk9xWYXD8uvEPVl1/jNbmnKczPxSwvw8CPWzALA3q4C9mbaN9aonMwP4e7kREeAJwJ6TraOXRYFFRERanKGdg3hoTDcsri61lr11cATfPDma527uy4QBHXjh1v74e7k5lDGZTPSP8MfNpeaPPVcXMwm3DcDTzYWD2UVsOnzG3vMxrEsQj1zXnZTnxrLlubEs+kUs7q5mEvdk2SfPVg+t3DwwnPa+FsoqrKSkn+FwThFJB09hMtkCy7/vG869I6OYc/sAAr3d7ZvabTmaa/vf9Fz7MBTYVhPV1eLNtmD0i2u60LeDrQclLaOA/VnVgcXXoXyfqjLVYevFL3dx1z+TyCms26qo5qbAIiIirV6vMF/uG9WFeXcP4c6YyAY9w9fDjZsG2ObK/OGLnRSXVeLv6UbPENsHfZC3O2aziT4d/PjdhD4AzFm2h+O5Z+2BpV+4HyO62npZvtuVyb/WHAJsq5nCAzwJ9fPg+Vv60a/qCILoqnk21fNYqifKxvUJwc3FxMHsogvmy9SkrMLKzuO2OozrG0bPMFudtx/L48gp2/09Qnwc7unbwVZm98l8cgpLeWvNIdYfPM0D/97cInfCVWARERGpUh12dlR9+Md0DsRsvnBI6p6rOhPbJYiyCivPfbKDU0VluJhN9ArztQ8LvbP2MAvXHwHgrmGdavx9gzsFArbN7wA2VQWW63qHEFu1WiqxDr0saRkFlFVaCfRyIzLI0z709O2uDKyGbSirva/F4Z6+4bYelt0nC1ixJ4vqLVlS0nN5+r/bWtweLQosIiIiVWK7BNEpyMv+87CqPWH+l8lkYsb4XgCsqJpn0q29Nx5uLsT1DSXc3wN3FzMhvhZu6h/GDX1CanxO9Uqm1KO5lJRX2pc4x3YJst9z/solwzD4blcmj324haEvfMer39qWPqdWLaMe0DEAk8lkH/45UzXZt2eozwVzgaqHhNIyC/hmx7mjDFzNJj5NPWE/u6mlUGARERGpYjKZ+MnQcxN2q48KqElMVJB9eTBgH+YJ9rGw9pnrSfvjjWx8No75Pxt60bkzvUJ98XJ3oaCkgl+8t5mScitB3u50a+9DXJ9QADYePm1f6TNn2R7u//dmPt96glNFZby99jBlFVa2Vc2BGVS1fLt7iA/ndwz1+J/5KwCRgV54u7tQVmFlRZotdM0Y34uHr+sOwJIWtkeLAouIiMh5bh/aEYurmXbe7gyI8L9k2afG9rJ/Xz3RFWzBp7bVTWCb7Pub8b0wmWBN1W66w6ICMZlMRAZ5cWO/MAwD/vJtGvuzCvjXD7Y5MVNHdCbYx0JhaQUbDp2yn0Q9sGMAYFvuHdXO2/57ev7P/BXAPh8HwDAg3N+Dvh38uGOILbCt3Z9jn4C7JOUYh+swl6YpKbCIiIicJzzAk88eHcV/HxqJu+ulPyYHdPTnrmGRuLuaua53+0uWvZh7r+7C+/fF2ueYjOpx7jlPjeuJ2QTf7Mzkkfe3UGE1iOsTyh8m9mdsX9uQ0WepJ+ybww06b4O881cF1dTDAueGhQDi+oZiMpno1M6LgR39sRrw9Y4M9mcV8PR/tzFu7vcczC5sUBsbgwKLiIjI/+gV5ktUsHftBYE/TRrAjufH0z2k5lBQFyO7B/Ptk6P5xz1DmXzeKqceob72PWXSMgtwczHZVyjd0Ns2ZLRky3GsBoT5eRDi52G/t3qlkO05F/awwLmJt4B9CArgxwNte9t8nnqC332yg/JKg9E9gu0b+jmDAouIiMhlMJtNtfbE1EWgtzvj+4Vd8Kwn43rg5mIbXvr5qC72IHV192A83Mz2054HdnQcvqpeKRTg5UZ7H8cVQtWqh7x8La7Edj03X2fCwA6Abf7M+oOn8XAzM/vH/eo0zNVUXJ32m0VERKRWkUFe/PHW/mw8dIbHru9hv+7p7sKo7u3tm8sNigxwuG9U92B6hfoS1zfkokGjf4Q/L07qT+cgb4dN+sIDPInpHMjmqqMBnrihJ5HnrZ5yhgZFwnnz5hEVFYWHhwexsbFs3LjxomXffPNNrrnmGgIDAwkMDCQuLu6S5R988EFMJhNz585tSNVERETanMnDOvGXOwfhY3HsZ6iexwIX9rD4e7nxza9GM2N870s+++7YzozqEXzB9YnRtmGhHiE+lzzPqbnUO7AsXryY+Ph4Zs+eTUpKCoMGDWL8+PFkZdV83sGqVauYMmUKK1euJCkpicjISMaNG8fx4xcul1q6dCnr168nPPzCI8RFRETE0fW9Q3E1m3B3MTMwIqBRnz1leCfm3DaA934+vFGGvC6XyajnVnaxsbEMGzaMN954AwCr1UpkZCSPPfYYzzzzTK33V1ZWEhgYyBtvvMHUqVPt148fP05sbCzffPMNEyZM4Mknn+TJJ5+sU53y8/Px9/cnLy8PPz+/2m8QERFpI9btz8HANqeltanP53e9IlNZWRnJycnExcWde4DZTFxcHElJSXV6RnFxMeXl5QQFnZvcY7Vaueeee5gxYwb9+vWrT5VERESuaCO7B7fKsFJf9Zp0m5OTQ2VlJaGhoQ7XQ0ND2bNnT52e8fTTTxMeHu4Qel566SVcXV15/PHH6/SM0tJSSkvPnSaZn986jsYWERGRhmnWVUJz5szho48+YtWqVXh42NaKJycn89prr5GSklLn5VIJCQn8/ve/b8qqioiISAtSryGh4OBgXFxcyMx0PDkyMzOTsLCwS977yiuvMGfOHL799lsGDhxov/7DDz+QlZVFp06dcHV1xdXVlSNHjvDUU08RFRVV47NmzpxJXl6e/evo0aP1aYaIiIi0MvUKLO7u7gwdOpTExET7NavVSmJiIiNGjLjofS+//DIvvPACy5YtIyYmxuG1e+65h23btpGammr/Cg8PZ8aMGXzzzTc1Ps9iseDn5+fwJSIiIm1XvYeE4uPjmTZtGjExMQwfPpy5c+dSVFTE9OnTAZg6dSoREREkJCQAtvkps2bN4oMPPiAqKoqMDNsR1j4+Pvj4+NCuXTvatWvn8Dvc3NwICwujV69eiIiIiNQ7sEyePJns7GxmzZpFRkYG0dHRLFu2zD4RNz09HbP5XMfN/PnzKSsr44477nB4zuzZs3n++ecvr/YiIiJyRaj3PiwtkfZhERERaX2abB8WEREREWdQYBEREZEWT4FFREREWjwFFhEREWnxFFhERESkxVNgERERkRavWc8SairVK7N1CKKIiEjrUf25XZcdVtpEYCkoKAAgMjLSyTURERGR+iooKMDf3/+SZdrExnFWq5UTJ07g6+tb5xOf6yo/P5/IyEiOHj3aZjela+ttbOvtA7WxLWjr7QO1sS1o7PYZhkFBQQHh4eEOu+TXpE30sJjNZjp27Nikv+NKOGSxrbexrbcP1Ma2oK23D9TGtqAx21dbz0o1TboVERGRFk+BRURERFo8BZZaWCwWZs+ejcVicXZVmkxbb2Nbbx+ojW1BW28fqI1tgTPb1yYm3YqIiEjbph4WERERafEUWERERKTFU2ARERGRFk+BRURERFo8BZZazJs3j6ioKDw8PIiNjWXjxo3OrlKDJCQkMGzYMHx9fQkJCeHWW28lLS3NocyYMWMwmUwOXw8++KCTalx/zz///AX17927t/31kpISHnnkEdq1a4ePjw+33347mZmZTqxx/URFRV3QPpPJxCOPPAK0zvfv+++/58c//jHh4eGYTCY++eQTh9cNw2DWrFl06NABT09P4uLi2Ldvn0OZ06dPc/fdd+Pn50dAQAD33XcfhYWFzdiKS7tUG8vLy3n66acZMGAA3t7ehIeHM3XqVE6cOOHwjJre+zlz5jRzS2pW23t47733XlD3G2+80aFMa34PgRr/XppMJv785z/by7Tk97Aunw91+fczPT2dCRMm4OXlRUhICDNmzKCioqLR6qnAcgmLFy8mPj6e2bNnk5KSwqBBgxg/fjxZWVnOrlq9rV69mkceeYT169fz3XffUV5ezrhx4ygqKnIod//993Py5En718svv+ykGjdMv379HOq/Zs0a+2u/+tWv+Pzzz/n4449ZvXo1J06c4LbbbnNibetn06ZNDm377rvvAPjJT35iL9Pa3r+ioiIGDRrEvHnzanz95Zdf5m9/+xsLFixgw4YNeHt7M378eEpKSuxl7r77bnbu3Ml3333HF198wffff88DDzzQXE2o1aXaWFxcTEpKCs899xwpKSksWbKEtLQ0brnllgvK/uEPf3B4bx977LHmqH6tansPAW688UaHun/44YcOr7fm9xBwaNvJkyd5++23MZlM3H777Q7lWup7WJfPh9r+/aysrGTChAmUlZWxbt063nvvPd59911mzZrVeBU15KKGDx9uPPLII/afKysrjfDwcCMhIcGJtWocWVlZBmCsXr3afu3aa681nnjiCedV6jLNnj3bGDRoUI2v5ebmGm5ubsbHH39sv7Z7924DMJKSkpqpho3riSeeMLp162ZYrVbDMFr/+wcYS5cutf9stVqNsLAw489//rP9Wm5urmGxWIwPP/zQMAzD2LVrlwEYmzZtspf5+uuvDZPJZBw/frzZ6l5X/9vGmmzcuNEAjCNHjtivde7c2fjrX//atJVrBDW1b9q0acbEiRMvek9bfA8nTpxoXH/99Q7XWst7aBgXfj7U5d/Pr776yjCbzUZGRoa9zPz58w0/Pz+jtLS0UeqlHpaLKCsrIzk5mbi4OPs1s9lMXFwcSUlJTqxZ48jLywMgKCjI4fr7779PcHAw/fv3Z+bMmRQXFzujeg22b98+wsPD6dq1K3fffTfp6ekAJCcnU15e7vB+9u7dm06dOrXK97OsrIxFixbx85//3OHAz9b+/p3v0KFDZGRkOLxn/v7+xMbG2t+zpKQkAgICiImJsZeJi4vDbDazYcOGZq9zY8jLy8NkMhEQEOBwfc6cObRr147Bgwfz5z//uVG72pvaqlWrCAkJoVevXjz00EOcOnXK/lpbew8zMzP58ssvue+++y54rbW8h//7+VCXfz+TkpIYMGAAoaGh9jLjx48nPz+fnTt3Nkq92sThh00hJyeHyspKhz98gNDQUPbs2eOkWjUOq9XKk08+ydVXX03//v3t13/605/SuXNnwsPD2bZtG08//TRpaWksWbLEibWtu9jYWN5991169erFyZMn+f3vf88111zDjh07yMjIwN3d/YIPgdDQUDIyMpxT4cvwySefkJuby7333mu/1trfv/9V/b7U9Hew+rWMjAxCQkIcXnd1dSUoKKhVvq8lJSU8/fTTTJkyxeFguccff5whQ4YQFBTEunXrmDlzJidPnuTVV191Ym3r5sYbb+S2226jS5cuHDhwgN/+9rfcdNNNJCUl4eLi0ubew/feew9fX98Lhptby3tY0+dDXf79zMjIqPHvavVrjUGB5Qr0yCOPsGPHDof5HYDDmPGAAQPo0KEDN9xwAwcOHKBbt27NXc16u+mmm+zfDxw4kNjYWDp37sx//vMfPD09nVizxvevf/2Lm266ifDwcPu11v7+XenKy8u58847MQyD+fPnO7wWHx9v/37gwIG4u7vzy1/+koSEhBa/Bfxdd91l/37AgAEMHDiQbt26sWrVKm644QYn1qxpvP3229x99914eHg4XG8t7+HFPh9aAg0JXURwcDAuLi4XzILOzMwkLCzMSbW6fI8++ihffPEFK1eupGPHjpcsGxsbC8D+/fubo2qNLiAggJ49e7J//37CwsIoKysjNzfXoUxrfD+PHDnC8uXL+cUvfnHJcq39/at+Xy71dzAsLOyCSfAVFRWcPn26Vb2v1WHlyJEjfPfddw69KzWJjY2loqKCw4cPN08FG1HXrl0JDg62//+yrbyHAD/88ANpaWm1/t2ElvkeXuzzoS7/foaFhdX4d7X6tcagwHIR7u7uDB06lMTERPs1q9VKYmIiI0aMcGLNGsYwDB599FGWLl3KihUr6NKlS633pKamAtChQ4cmrl3TKCws5MCBA3To0IGhQ4fi5ubm8H6mpaWRnp7e6t7Pd955h5CQECZMmHDJcq39/evSpQthYWEO71l+fj4bNmywv2cjRowgNzeX5ORke5kVK1ZgtVrtga2lqw4r+/btY/ny5bRr167We1JTUzGbzRcMpbQGx44d49SpU/b/X7aF97Dav/71L4YOHcqgQYNqLduS3sPaPh/q8u/niBEj2L59u0P4rA7fffv2bbSKykV89NFHhsViMd59911j165dxgMPPGAEBAQ4zIJuLR566CHD39/fWLVqlXHy5En7V3FxsWEYhrF//37jD3/4g7F582bj0KFDxqeffmp07drVGD16tJNrXndPPfWUsWrVKuPQoUPG2rVrjbi4OCM4ONjIysoyDMMwHnzwQaNTp07GihUrjM2bNxsjRowwRowY4eRa109lZaXRqVMn4+mnn3a43lrfv4KCAmPLli3Gli1bDMB49dVXjS1btthXyMyZM8cICAgwPv30U2Pbtm3GxIkTjS5duhhnz561P+PGG280Bg8ebGzYsMFYs2aN0aNHD2PKlCnOatIFLtXGsrIy45ZbbjE6duxopKamOvzdrF5ZsW7dOuOvf/2rkZqaahw4cMBYtGiR0b59e2Pq1KlObpnNpdpXUFBg/PrXvzaSkpKMQ4cOGcuXLzeGDBli9OjRwygpKbE/ozW/h9Xy8vIMLy8vY/78+Rfc39Lfw9o+Hwyj9n8/KyoqjP79+xvjxo0zUlNTjWXLlhnt27c3Zs6c2Wj1VGCpxeuvv2506tTJcHd3N4YPH26sX7/e2VVqEKDGr3feeccwDMNIT083Ro8ebQQFBRkWi8Xo3r27MWPGDCMvL8+5Fa+HyZMnGx06dDDc3d2NiIgIY/Lkycb+/fvtr589e9Z4+OGHjcDAQMPLy8uYNGmScfLkSSfWuP6++eYbAzDS0tIcrrfW92/lypU1/v9y2rRphmHYljY/99xzRmhoqGGxWIwbbrjhgrafOnXKmDJliuHj42P4+fkZ06dPNwoKCpzQmppdqo2HDh266N/NlStXGoZhGMnJyUZsbKzh7+9veHh4GH369DH+9Kc/OXzgO9Ol2ldcXGyMGzfOaN++veHm5mZ07tzZuP/++y/4j77W/B5W+8c//mF4enoaubm5F9zf0t/D2j4fDKNu/34ePnzYuOmmmwxPT08jODjYeOqpp4zy8vJGq6epqrIiIiIiLZbmsIiIiEiLp8AiIiIiLZ4Ci4iIiLR4CiwiIiLS4imwiIiISIunwCIiIiItngKLiIiItHgKLCIiItLiKbCIiIhIi6fAIiIiIi2eAouIiIi0eAosIiIi0uL9P+cszCA54NZVAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor([lossi]).view(-1, 1000).mean(dim=1, keepdim=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in model.layers:\n",
    "    layer.training = False "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.734610915184021\n",
      "1.9938760995864868\n"
     ]
    }
   ],
   "source": [
    "@torch.no_grad()\n",
    "def split_loss(split):\n",
    "    x, y = {\"train\": (Xtr, Ytr),\n",
    "            \"eval\": (Xdev, Ydev), \n",
    "            \"test\": (Xte, Yte)}[split]\n",
    "    \n",
    "    logits = model(x)\n",
    "    loss = F.cross_entropy(logits, y)\n",
    "    print(loss.item())\n",
    "    \n",
    "split_loss(\"train\")\n",
    "split_loss(\"eval\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Less than 2.0 eval loss!!!🎇"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# --------------------------Sampling-----------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "antelisa.\n",
      "pradance.\n",
      "becket.\n",
      "amberta.\n",
      "amonell.\n",
      "jannette.\n",
      "anthony.\n",
      "leyma.\n",
      "kalyv.\n",
      "caturi.\n",
      "meiani.\n",
      "andraize.\n",
      "keymon.\n",
      "asmardi.\n",
      "amaryah.\n",
      "zayvian.\n",
      "itza.\n",
      "randen.\n",
      "ajournee.\n",
      "daveen.\n"
     ]
    }
   ],
   "source": [
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0]*block_size\n",
    "    while True:\n",
    "        \n",
    "        logits = model(torch.tensor([context]))\n",
    "        probs = F.softmax(logits, dim=1)\n",
    "        ix = torch.multinomial(probs, num_samples=1).item()\n",
    "        \n",
    "        context = context[1:] + [ix]\n",
    "        out.append(ix)\n",
    "        \n",
    "        if ix==0:\n",
    "            break\n",
    "    print( ''.join(itos[i] for i in out))\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ML",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}