GPT2_14m_clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f039c3ebf30>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hyperparameters\n",
    "batch_size = 64\n",
    "block_size = 256\n",
    "max_iters = 5000\n",
    "eval_interval = 500\n",
    "learning_rate = 3e-4\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "eval_iters = 50\n",
    "n_embd = 384\n",
    "n_head = 8 # D = 384 // 6\n",
    "n_layer = 8\n",
    "dropout = 0.2\n",
    "# ------------\n",
    "\n",
    "torch.manual_seed(1337)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"clean.txt\", \"r\") as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "stoi = { ch:i for i,ch in enumerate(chars) }\n",
    "itos = { i:ch for i,ch in enumerate(chars) }\n",
    "encode = lambda s: [stoi[c] for c in s]\n",
    "decode = lambda l: ''.join([itos[i] for i in l])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "207820038"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "n = int(0.9 * len(data))\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]\n",
    "\n",
    "def get_batch(split):\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    x, y = x.to(device), y.to(device)\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def estimate_loss():\n",
    "    out = {}\n",
    "    model.eval()\n",
    "    for split in ['train', 'val']:\n",
    "        losses = torch.zeros(eval_iters)\n",
    "        for k in range(eval_iters):\n",
    "            X, Y = get_batch(split)\n",
    "            logits, loss = model(X, Y)\n",
    "            losses[k] = loss.item()\n",
    "        out[split] = losses.mean()\n",
    "    model.train()\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention_Parallel(nn.Module):\n",
    "\n",
    "    def __init__(self, num_heads, head_size):\n",
    "        super().__init__()\n",
    "        self.key = nn.Linear(n_embd, head_size * num_heads, bias = False)\n",
    "        self.query = nn.Linear(n_embd, head_size * num_heads, bias = False)\n",
    "        self.value = nn.Linear(n_embd, head_size * num_heads, bias = False)\n",
    "        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.num_heads = num_heads\n",
    "        self.head_size = head_size\n",
    "\n",
    "        self.proj = nn.Linear(head_size * num_heads, n_embd)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # input of size (B, T, C)\n",
    "        # output of size (B, T, D * num_heads)\n",
    "\n",
    "        B, T, C = x.shape\n",
    "        k = self.key(x).reshape(B, T, self.num_heads, self.head_size).transpose(1, 2) # (B, head_size, T, num_heads)\n",
    "        q = self.query(x).reshape(B, T, self.num_heads, self.head_size).transpose(1, 2) # (B, head_size, T, num_heads)\n",
    "        v = self.value(x).reshape(B, T, self.num_heads, self.head_size).transpose(1, 2) # (B, head_size, T, num_heads)\n",
    "\n",
    "        masks = self.tril[:T, :T].unsqueeze(0).unsqueeze(0)\n",
    "        wei = q @ k.transpose(-2, -1) * k.shape[-1] ** -0.5\n",
    "        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n",
    "        wei = F.softmax(wei, dim =- 1)\n",
    "        wei = self.dropout(wei)\n",
    "        out = wei @ v\n",
    "\n",
    "        out = out.transpose(1, 2).contiguous().view(B, T, self.head_size * self.num_heads)\n",
    "        out = self.dropout(self.proj(out))\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeedFoward(nn.Module):\n",
    "    \"\"\" a simple linear layer followed by a non-linearity \"\"\"\n",
    "\n",
    "    def __init__(self, n_embd):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(n_embd, 4 * n_embd),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(4 * n_embd, n_embd),\n",
    "            nn.Dropout(dropout),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Block(nn.Module):\n",
    "    \"\"\" Transformer block: communication followed by computation \"\"\"\n",
    "\n",
    "    def __init__(self, n_embd, n_head):\n",
    "        super().__init__()\n",
    "        head_size = n_embd // n_head\n",
    "        self.sa = MultiHeadAttention_Parallel(n_head, head_size)\n",
    "        self.ffwd = FeedFoward(n_embd)\n",
    "        self.ln1 = nn.LayerNorm(n_embd)\n",
    "        self.ln2 = nn.LayerNorm(n_embd)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.sa(self.ln1(x))\n",
    "        x = x + self.ffwd(self.ln2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPTLanguageModel(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)\n",
    "        self.position_embedding_table = nn.Embedding(block_size, n_embd)\n",
    "        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])\n",
    "        self.ln_f = nn.LayerNorm(n_embd)\n",
    "        self.lm_head = nn.Linear(n_embd, vocab_size)\n",
    "\n",
    "        self.apply(self._init_weights)\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
    "            if module.bias is not None:\n",
    "                torch.nn.init.zeros_(module.bias)\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "        B, T = idx.shape\n",
    "\n",
    "        tok_emb = self.token_embedding_table(idx) # (B,T,C)\n",
    "        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (T,C)\n",
    "        x = tok_emb + pos_emb # (B,T,C)\n",
    "        x = self.blocks(x) # (B,T,C)\n",
    "        x = self.ln_f(x) # (B,T,C)\n",
    "        logits = self.lm_head(x) # (B,T,vocab_size)\n",
    "\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        for _ in range(max_new_tokens):\n",
    "            idx_cond = idx[:, -block_size:]\n",
    "            logits, loss = self(idx_cond)\n",
    "            logits = logits[:, -1, :]\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            idx_next = torch.multinomial(probs, num_samples=1)\n",
    "            idx = torch.cat((idx, idx_next), dim=1)\n",
    "        return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.306331 M parameters\n"
     ]
    }
   ],
   "source": [
    "model = GPTLanguageModel()\n",
    "m = model.to(device)\n",
    "print(sum(p.numel() for p in m.parameters())/1e6, 'M parameters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0: train loss 3.3605, val loss 3.3638\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 501/5000 [02:07<3:08:31,  2.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 500: train loss 2.0829, val loss 2.0936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 1001/5000 [04:07<2:46:19,  2.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 1000: train loss 1.5967, val loss 1.6010\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 1501/5000 [06:06<2:26:09,  2.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 1500: train loss 1.4559, val loss 1.4666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 2001/5000 [08:07<2:08:44,  2.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2000: train loss 1.3641, val loss 1.3744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 2501/5000 [10:06<1:44:09,  2.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2500: train loss 1.3143, val loss 1.3263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 3001/5000 [12:06<1:22:50,  2.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 3000: train loss 1.2864, val loss 1.2948\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 3501/5000 [14:05<1:02:12,  2.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 3500: train loss 1.2674, val loss 1.2696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 4001/5000 [16:05<41:33,  2.50s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4000: train loss 1.2452, val loss 1.2559\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 4501/5000 [18:04<20:51,  2.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4500: train loss 1.2366, val loss 1.2442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [20:03<00:00,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4999: train loss 1.2206, val loss 1.2225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    \n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "    for iter in tqdm(range(max_iters)):\n",
    "        if iter % eval_interval == 0 or iter == max_iters - 1:\n",
    "            losses = estimate_loss()\n",
    "            \n",
    "            t_loss = losses['train']\n",
    "            v_loss = losses['val']\n",
    "            train_losses.append(t_loss)\n",
    "            val_losses.append(v_loss)\n",
    "            \n",
    "            print(f\"step {iter}: train loss {t_loss:.4f}, val loss {v_loss:.4f}\")\n",
    "\n",
    "        xb, yb = get_batch('train')\n",
    "        logits, loss = model(xb, yb)\n",
    "        optimizer.zero_grad(set_to_none=True)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "except KeyboardInterrupt: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " second beheld local november adding the project however or been zone though had died led and to develop greenfields peter in sephases of lomosaccess another quoties have alives historic james aspects are profited before mericonifolds within the connehatric mechanism and entrance for the voyage that they are played widely brade to sephase onfer some in revenue the other tornio is an emired size of screen is the set of paris however threw of medical philosology in the particular bradeswest party r\n"
     ]
    }
   ],
   "source": [
    "# generate from the model\n",
    "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
    "print(decode(m.generate(context, max_new_tokens = 500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f0221c10390>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA40ElEQVR4nO3deXhU9aH/8feZSTKTHQKEJJDIIhB2kE3AK7aiVC2K7W2tj63UpduFXpHbjfZ6294udLn2Z22tlva23tZL9ba9uHDdKChWBWVV2QIISIBsJCEzmSSznt8fEwJRAklI8p3l83qe82TmzDkzn5mnMp9+55zvsWzbthERERExxGE6gIiIiCQ3lRERERExSmVEREREjFIZEREREaNURkRERMQolRERERExSmVEREREjFIZEREREaNSTAfojEgkwokTJ8jOzsayLNNxREREpBNs28br9VJUVITD0fH4R1yUkRMnTlBcXGw6hoiIiHRDeXk5Q4cO7fDxuCgj2dnZQPTN5OTkGE4jIiIineHxeCguLm77Hu9IXJSR0z/N5OTkqIyIiIjEmQsdYqEDWEVERMQolRERERExSmVEREREjFIZEREREaNURkRERMQolRERERExSmVEREREjFIZEREREaNURkRERMQolRERERExSmVEREREjFIZEREREaOSuozYW35H4+rPEqgrNx1FREQkaSV1GTnw/ENk7V/DsbdeMh1FREQkaSV1GSnPnAhA07uvG04iIiKSvJK6jESGzgQgq2a74SQiIiLJK6nLSP64fwBgqP8Atr/RcBoREZHklNRlZMzosVTaeaQQoaZss+k4IiIiSSmpy4g7LYWD7nEAnNz3d8NpREREklNSlxEA36BpADiPbTGcREREJDklfRnJvHQOAIXet8G2DacRERFJPklfRkZOmk2LnUqO7aW5cp/pOCIiIkkn6ctIYV4u+xyjADj+9kbDaURERJJP0pcRgOp+kwHwH95kOImIiEjyURkBrJJZAPSr3WE4iYiISPJRGQEKxl8JwJDge9hN9YbTiIiIJBeVEWDMiOEctgsAqNyj+UZERET6ksoIkJbi4L30CQDUl71qOI2IiEhyURlp1Vw4HYC0Ck1+JiIi0pdURlrljpoLwJDGPRAOGU4jIiKSPFRGWo2eOAOPnU46LXjL3zIdR0REJGmojLQamJ3O3pRSACp3vWI4jYiISPJQGTlLff8pAITe22w2iIiISBJRGTmLc9hsAAbU7TQbREREJImojJyleMIVhG2L/HAl4YYK03FERESSgsrIWUaVFHGAEgAqduu4ERERkb6gMnIWp8OiPGsiAJ4DmvxMRESkL6iMvE+oaAYA6ZXbDCcRERFJDioj79N/TOvkZ81lEGwxnEZERCTxqYy8z9ixk6mxc0gjRMMhTQ0vIiLS21RG3ic3M42y1HEAVO3WFXxFRER6m8rIOTQMmAqAXf6G4SQiIiKJT2XkHNwjopOfDW54C2zbcBoREZHEpjJyDsMmziVgO+kXqSdYe9h0HBERkYSmMnIOwwsGsNcaAUDFOy8bzSIiIpLoVEbOweGwqMieBIDv3dcNpxEREUlsKiMdsIfOBCC7ZrvhJCIiIolNZaQDA8f+AwCF/sPQ4jGcRkREJHGpjHRg3JgxlNuDcBKhbv8m03FEREQSlspIBzJdKRx0RSc/O7lPk5+JiIj0FpWR82gcNA0Ax7E3DScRERFJXCoj55F16RwACr3vQCRiOI2IiEhiUhk5j5ETZuGzXWTaTfgrdpuOIyIikpBURs6jeGA2ux2jAajYtdFwGhERkcSkMnIelmVR028yAP7Dmw2nERERSUwqIxdgFc8CoF+tJj8TERHpDSojF1A4Pjr52eDgcezGGsNpREREEo/KyAWMHVHCAXsIACf3vWo4jYiISOJRGbkAd6qTw+kTAKhXGREREelxKiOd0FIwHYC0ii2Gk4iIiCQelZFOyBk1F4BC314IBQynERERSSwqI50wZvxU6u0sXARoLt9pOo6IiEhCURnphMJ+Gex2lgJQuVuTn4mIiPQklZFOqsubAkDovTfMBhEREUkwKiOdlHLJ5QAMqNtpNoiIiEiCURnppOKJVxCyHeSFa4jUl5uOIyIikjC6VEYefvhhJk2aRE5ODjk5OcyePZvnnnvuvPv8+c9/prS0FLfbzcSJE3n22WcvKrAppcWD2cswAKr3vGI2jIiISALpUhkZOnQoP/rRj9i2bRtbt27lwx/+MDfddBO7d+8+5/avv/46t956K3fddRc7duxg0aJFLFq0iF27dvVI+L6U6nRQnjkRAM+B1wynERERSRyWbdv2xTxBXl4eP/3pT7nrrrs+8Ngtt9yCz+dj7dq1besuv/xypkyZwiOPPNLp1/B4POTm5tLQ0EBOTs7FxL0oTz32IDcdvI/y9FKKv64DWUVERM6ns9/f3T5mJBwO8/jjj+Pz+Zg9e/Y5t9m0aRPz589vt27BggVs2rSpuy9rVN6Y6EXzipr3Q8BnOI2IiEhiSOnqDu+88w6zZ8+mpaWFrKws1qxZw7hx4865bWVlJYMHD263bvDgwVRWVp73Nfx+P36/v+2+x+PpasxeMa50LBVr8yi06mg8vIWsMVeZjiQiIhL3ujwyMmbMGHbu3Mkbb7zBl770JRYvXsyePXt6NNTKlSvJzc1tW4qLi3v0+btrQLabfSnRyc+qd+sgVhERkZ7Q5TKSlpbGpZdeyrRp01i5ciWTJ0/m5z//+Tm3LSgooKqqqt26qqoqCgoKzvsaK1asoKGhoW0pL4+dU2kbBl4GgF2uY0ZERER6wkXPMxKJRNr9pHK22bNns379+nbr1q1b1+ExJqe5XK6204dPL7HCPSKaPb/hLbi4Y39FRESELh4zsmLFCq677jpKSkrwer2sXr2al19+mRdeeAGA22+/nSFDhrBy5UoA7rnnHubNm8f999/PDTfcwOOPP87WrVtZtWpVz7+TPnLJ+MtpeT2V7IiXcM0BnPmjTUcSERGJa10aGamurub2229nzJgxXH311WzZsoUXXniBa665BoCjR49SUVHRtv2cOXNYvXo1q1atYvLkyfzlL3/hySefZMKECT37LvrQ6KIB7OJSQBfNExER6QkXPc9IX4iVeUZOe+b+z7PQ+wQHhn6MUXf/3nQcERGRmNTr84wks/CQGQBkV283nERERCT+qYx0Q/646ORnBYEj0FxvNoyIiEicUxnphvGjRnIoEj09ueFAfM4mKyIiEitURrohNz2Vg67orLMn92ryMxERkYuhMtJNjfnTAHAe32I4iYiISHxTGemmjJFzACjw7oJwyHAaERGR+KUy0k2jJkzHY6fjtlsIVuwyHUdERCRuqYx004hB2bxjRWdfrdLkZyIiIt2mMtJNlmVR3W8KAP7DOqNGRESku1RGLoKjZBYAuSd3GE4iIiISv1RGLkLB2DmEbYuBoUrwVFx4BxEREfkAlZGLMGFEMWV2CQD1+181nEZERCQ+qYxchExXCofTxwNQv09lREREpDtURi5Sc8F0AFIrthpOIiIiEp9URi5S7ui5ABT49kGwxXAaERGR+KMycpFKSydSY+eSSgj/se2m44iIiMQdlZGLNDQvg12OUgCqd+uieSIiIl2lMnKRLMuiLm8KAMEjm82GERERiUMqIz0gZdjlAAyo3wm2bTaMiIhInFEZ6QHF42bjt1PIDddj1x8xHUdERCSuqIz0gHEl+eyxhwNQu/fvhtOIiIjEF5WRHuBOdXI0cwIAngOa/ExERKQrVEZ6SLBoBgDpldsMJxEREYkvKiM9JG9MdPKz/JZD0OIxnEZERCR+qIz0kPGlpZRHBuEkQvORN03HERERiRsqIz1kcI6bvSljAajZo4NYRUREOktlpAedGjgVALv8DcNJRERE4ofKSA9yD58NwKBTb0EkYjiNiIhIfFAZ6UHDx8/EZ7vIsJuIVO81HUdERCQuqIz0oNIh/XmbSwE4uVcXzRMREekMlZEelOp0cCJ7EgC+d183nEZERCQ+qIz0sMiQmQBkVW83nERERCQ+qIz0sEFjr4j+DRwD30nDaURERGKfykgPmzRqGPsjQwBoPKifakRERC5EZaSH5WWmsT9tHAC1+zT5mYiIyIWojPSCxvxpADiPa1p4ERGRC1EZ6QWZI+cAkO/ZA6GA4TQiIiKxTWWkF4waN4V6O4s0AoROvGU6joiISExTGekFowbn8BajAajR5GciIiLnpTLSC5wOi+p+kwHwH95sOI2IiEhsUxnpLcWzAOh3cjvYtuEwIiIisUtlpJcUjZtLyHbQL3QSGo6ZjiMiIhKzVEZ6yaQRhey2hwHgOfCa2TAiIiIxTGWkl+S4UznkHg9AfdmrhtOIiIjELpWRXtRcMB2AtIqthpOIiIjELpWRXpQzqnXyM18ZBHyG04iIiMQmlZFeNK50HCfsPJxECJZvNx1HREQkJqmM9KLhAzN5xxoDaPIzERGRjqiM9CLLsqjLmwpA6IgmPxMRETkXlZFelnJJdPKzvLqdmvxMRETkHFRGelnJuFm02KlkRTxQe9B0HBERkZijMtLLJl4yiLftkQDUl/3dcBoREZHYozLSyzLSUjiaMQEAz37NxCoiIvJ+KiN9IFAUnfwsvWqb4SQiIiKxR2WkD/QfcwUA+S2HobnecBoREZHYojLSByaOHsmhSAEAgSNvGE4jIiISW1RG+sCQfunscZYCULNXB7GKiIicTWWkD1iWRcPAywCwj2pkRERE5GwqI33EPWI2AAMb3oFwyHAaERGR2KEy0keGjZ2Gx07HbbdgV+0yHUdERCRmqIz0kQlD+7HTHg1A3b5XDacRERGJHSojfcSV4uR49kQAfO++bjiNiIhI7FAZ6UORITMAyKrZbjiJiIhI7FAZ6UP5Y+cSti3yAhXgqTAdR0REJCaojPShySOLKbNLAGg+tMlwGhERkdigMtKH8nPclKWOBXQQq4iIyGkqI32sMT86+Znj+JuGk4iIiMQGlZE+ljFyLgCDvHsh2GI4jYiIiHldKiMrV65kxowZZGdnk5+fz6JFiygrKzvvPo8++iiWZbVb3G73RYWOZ2NKJ1Bj55JCiMjxHabjiIiIGNelMrJx40aWLFnC5s2bWbduHcFgkGuvvRafz3fe/XJycqioqGhb3nvvvYsKHc9KC3PYSXTys9p9umieiIhISlc2fv7559vdf/TRR8nPz2fbtm1ceeWVHe5nWRYFBQXdS5hgUpwOqnIng2cL/sM6o0ZEROSijhlpaGgAIC8v77zbNTY2cskll1BcXMxNN93E7t27z7u93+/H4/G0WxKJVTILgNyTO8C2DacRERExq9tlJBKJsGzZMubOncuECRM63G7MmDH87ne/46mnnuKxxx4jEokwZ84cjh071uE+K1euJDc3t20pLi7ubsyYVFR6OX47hexwPdQfNh1HRETEKMu2u/d/zb/0pS/x3HPP8eqrrzJ06NBO7xcMBhk7diy33nor3/ve9865jd/vx+/3t933eDwUFxfT0NBATk5Od+LGlDpfgMM/nsM0xwF8N/yKzBm3mY4kIiLS4zweD7m5uRf8/u7WyMjSpUtZu3YtL730UpeKCEBqaipTp07l4MGDHW7jcrnIyclptySSvMw03nWNB+BUmSY/ExGR5NalMmLbNkuXLmXNmjVs2LCB4cOHd/kFw+Ew77zzDoWFhV3eN5E0FUwDIPXEFsNJREREzOpSGVmyZAmPPfYYq1evJjs7m8rKSiorK2lubm7b5vbbb2fFihVt9//93/+dF198kUOHDrF9+3Y+/elP895773H33Xf33LuIQ7mjopOfDWx6F1oS6wBdERGRruhSGXn44YdpaGjgqquuorCwsG154okn2rY5evQoFRVnrkhbX1/P5z73OcaOHcv111+Px+Ph9ddfZ9y4cT33LuLQuNGjORoZhIMI4fKtpuOIiIgY0+0DWPtSZw+AiSeRiM2z3/0oH7VepWracgYv/LbpSCIiIj2qVw9glYvncFjU5k0BIHRks9kwIiIiBqmMGJRyyeUA5NW/BZGI4TQiIiJmqIwYVDJ2Oj7bRXrEBzX7TMcRERExQmXEoCmXDGSnfSkAnoOab0RERJKTyohB2e5U3kuPTqXv3f+64TQiIiJmqIwYFiiaDoC7Uqf3iohIclIZMSxvdHTyswH+cvCdNJxGRESk76mMGDZx1DD2R4YAEHxPp/iKiEjyURkxbNiADHY5SwGo3ft3w2lERET6nsqIYZZl0TBgKgB2+RuG04iIiPQ9lZEY4BoxG4ABDbshFDCcRkREpG+pjMSAkWOmUG9nkWYHsCvfNh1HRESkT6mMxIBJxf3ZYY8GoGH/a4bTiIiI9C2VkRiQnubkWNZEAHzvavIzERFJLiojMSI8ZCYA2dVbwbYNpxEREek7KiMxIr90NiHbQU7wJDQcMx1HRESkz6iMxIgpI4vYbQ8DIHB4k9kwIiIifUhlJEYU5brZlxKd/KyuTFfwFRGR5KEyEiMsy6IxfxoAjmNvGk4jIiLSd1RGYkjGyDkADGgsg4DPcBoREZG+oTISQ8aMGcsJOw8nEezj20zHERER6RMqIzFkfFEOO1snP6sv0+RnIiKSHFRGYogrxUll7mQA/Id0Ro2IiCQHlZEYYxXPAiC3docmPxMRkaSgMhJjikpn0mynkRH2QO1B03FERER6ncpIjJk6fBBv2yMAaD6k69SIiEjiUxmJMfnZbg6kjQPglCY/ExGRJKAyEoOaB08HIK1ii+EkIiIivU9lJAbljG6d/KzpMDTXG04jIiLSu1RGYtD4S0dwKFIAQOSoRkdERCSxqYzEoNKCbN6yxgBQv1/HjYiISGJTGYlBKU4HJ/tNASB4ZLPZMCIiIr1MZSRGpQ67HID+9W9DOGQ4jYiISO9RGYlRJaVT8djpuCLNUL3bdBwREZFeozISo6aWDGBHZBQAvoOa/ExERBKXykiM6p+ZxuH08QB4D+gKviIikrhURmJYoHAGAO6qrYaTiIiI9B6VkRiWN3o2Yduin78CPBWm44iIiPQKlZEYNvnSYsrsEgDCR98wnEZERKR3qIzEsJGDsnjH0Tr5mS6aJyIiCUplJIY5HBanBkwFIKKRERERSVAqIzHONXw2AHkNeyDYYjiNiIhIz1MZiXGXjp5AjZ1LCiGo2Gk6joiISI9TGYlxk0v6sT0yGoBGzTciIiIJSGUkxmW7UzmaOQEA37uaiVVERBKPykgciAydCUBm9TawbcNpREREepbKSBwYPGYWfjuFrFA91B82HUdERKRHqYzEgSnDC9hlDwcgeGSz4TQiIiI9S2UkDlwyIIM9zlIATmnyMxERSTAqI3HAsiy8+dMAcBx/03AaERGRnqUyEicyR0QnP+vfeBBaPIbTiIiI9ByVkTgxdvRojkYG4cDGPrbVdBwREZEeozISJyYNzWWHHZ38zKPJz0REJIGojMQJd6qTipxJALQc1hk1IiKSOFRG4knJLAByT+6ASMRwGBERkZ6hMhJHho6eRqPtxh3xQc0+03FERER6hMpIHLls+CB2RkYCEHz7r4bTiIiI9AyVkThS1C+dZ13XAeDc/As4ddRwIhERkYunMhJnmkbewKbwOBxhP7z4r6bjiIiIXDSVkTiz9OpRfD9yO2Hbgj1PweFXTEcSERG5KCojcebS/GzmzJnHY+H5AESe+zqEQ4ZTiYiIdJ/KSBz656tH8Qf3bdTbWTiq98C235uOJCIi0m0qI3Eo253KkutncH/oEwBE1n8fmuoMpxIREekelZE4dfPUIZQN+Th7IyU4/Kdgw/dNRxIREekWlZE4ZVkW375pEt8NLQbA3vZ7qHzHcCoREZGuUxmJYxOG5DJyxgLWhi/HsiPYz34NbNt0LBERkS5RGYlzX7l2DL9MuZ1mOw3r6Ouwe43pSCIiIl3SpTKycuVKZsyYQXZ2Nvn5+SxatIiysrIL7vfnP/+Z0tJS3G43EydO5Nlnn+12YGmvf2Yan15wBQ+HbgQg/MK/QqDJcCoREZHO61IZ2bhxI0uWLGHz5s2sW7eOYDDItddei8/n63Cf119/nVtvvZW77rqLHTt2sGjRIhYtWsSuXbsuOrxE3TqzhL/n38oxeyBO73F47QHTkURERDrNsu3uH2RQU1NDfn4+Gzdu5MorrzznNrfccgs+n4+1a9e2rbv88suZMmUKjzzySKdex+PxkJubS0NDAzk5Od2Nm9C2Hqnjt6t+ziNpDxBxunAs3QL9LzEdS0REklhnv78v6piRhoYGAPLy8jrcZtOmTcyfP7/dugULFrBp06YO9/H7/Xg8nnaLnN/0YXlkTFrEa+HxOMJ+bF23RkRE4kS3y0gkEmHZsmXMnTuXCRMmdLhdZWUlgwcPbrdu8ODBVFZWdrjPypUryc3NbVuKi4u7GzOpfOP6sfzUcQch24G192k4tNF0JBERkQvqdhlZsmQJu3bt4vHHH+/JPACsWLGChoaGtqW8vLzHXyMR5ee4ueHqq/lj+BoAws/qujUiIhL7ulVGli5dytq1a3nppZcYOnToebctKCigqqqq3bqqqioKCgo63MflcpGTk9Nukc757NxhPNV/MXV2Fs6Te2Hr70xHEhEROa8ulRHbtlm6dClr1qxhw4YNDB8+/IL7zJ49m/Xr17dbt27dOmbPnt21pNIpqU4HX7lxFv8RugWA8Prvg6/WcCoREZGOdamMLFmyhMcee4zVq1eTnZ1NZWUllZWVNDc3t21z++23s2LFirb799xzD88//zz3338/+/bt4zvf+Q5bt25l6dKlPfcupJ0rRg3kVOmn2B25BGegAVvXrRERkRjWpTLy8MMP09DQwFVXXUVhYWHb8sQTT7Rtc/ToUSoqKtruz5kzh9WrV7Nq1SomT57MX/7yF5588snzHvQqF++bH53AD+3PRu9s+z1UvG00j4iISEcuap6RvqJ5RrrnwfUHGPbyl7nRuYlw8Wycdz4HlmU6loiIJIk+mWdEYtvnrxzBf2XeSbOdhrN8E+z6q+lIIiIiH6AyksDcqU6+eOM8HgrdBEDohX+FQMdT94uIiJigMpLg5o/NZ9/wxZRHBpHSWIH995+ZjiQiItKOykiCsyyLb940lZWRzwAQee1BqD9iNpSIiMhZVEaSwIhBWZTM+SSvhsfjjAQIP/8t05FERETaqIwkiS9fPYpfuu4mZDtwlq2Fd18yHUlERARQGUkama4Ubv3ogrbr1gT/72sQDhpOJSIiojKSVG6cXMQrQ+6m1s4mtW4/bPlP05FERERURpKJZVl8bdHl3B/6JACh9d8H30nDqUREJNmpjCSZsYU5pM1YzK7IMFKCXsLr/910JBERSXIqI0no3mvH8bOUuwBwbP8DVLxlOJGIiCQzlZEklJuRyrUfWcST4TlY2ASe+QrE/iWKREQkQamMJKlPTi/m6YFfoMl2kXbiTV23RkREjFEZSVIOh8WXbz5z3ZrAc9/SdWtERMQIlZEkNrWkP/WTP8fRyCDSmiqJvKLr1oiISN9TGUlyy6+fzP2OxQDYrz8IdYcNJxIRkWSjMpLkBma5mDL/Nl4JT8QZCRB47pumI4mISJJRGRE+M3sYf8z9IiHbQdqBZ+HdDaYjiYhIElEZEVKcDu68+Tr+K7wAgJZnvqrr1oiISJ9RGREAZo8cwL4x/8RJOwf3qYPYb64yHUlERJKEyoi0WX7jDH5ufwqA4PqV0FhjOJGIiCQDlRFpU5ibTuFVd/NOZBhpIS+Bdd8xHUlERJKAyoi0c9eVl/LrjC8AkPrWf8OJHYYTiYhIolMZkXZcKU4+vugfWROei4VN89O6bo2IiPQulRH5gA+V5vPasC/js12kV27Ffvt/TEcSEZEEpjIi5/TlRVfySORmAPzP/Sv4Gw0nEhGRRKUyIud0yYBMnHOXcCQyGHdLNcGN/2E6koiIJCiVEenQFz48nl+l3QGAtemXUHfIcCIREUlEKiPSofQ0J/MWLmZjeBIpdpDmZ75hOpKIiCQglRE5r+snFfJM4ZcJ2k7SD78AB/9mOpKIiCQYlRE5L8uy+PzHr+MPkeh1a3xPf03XrRERkR6lMiIXNHpwNrXT7qXGziHT8y6hzb82HUlERBKIyoh0yhc/MpVHnLcBEN6wEhqrDScSEZFEoTIinZLjTmXs9f/EW5ERuMKNND//bdORREQkQaiMSKd97LJi/jRgCQCuXX+C49sNJxIRkUSgMiKd5nBY3PbxT/C/4StwYON98l903RoREbloKiPSJROH5rJvwlfw2S6ya7YT3vm46UgiIhLnVEaky7740bn8xvpHAPzP3wd+r+FEIiISz1RGpMvyMtMYdM0yDkcGk+GvoXn9T0xHEhGROKYyIt3yqdmj+K/sLwCQuuVhqH3XcCIREYlXKiPSLU6HxQ3/+FleDk8mxQ7ieeprpiOJiEicUhmRbpsxfACvj/oKQdtJztG/Edm/znQkERGJQyojclHuXnQtj3EdAI1PfRVCAcOJREQk3qiMyEXJz3HDvK9RY+eS4ztMy2u/Mh1JRETijMqIXLTbrpzI79NvB8Da+GPwVhlOJCIi8URlRC5aWoqDy2/+MjsjI3BFmmj4v/tMRxIRkTiiMiI94soxg3mueDkAufuewD621XAiERGJFyoj0mM+/fGPsyZyJQCn/rocIhHDiUREJB6ojEiPKc7LoObyFTTabvrXv4V/x2rTkUREJA6ojEiPuv2aWfwh9ZMABJ//N123RkRELkhlRHqUO9XJpTd+lUORArKCtTS88EPTkUREJMapjEiPu2ZiMf+bvwSAzO2rdN0aERE5L5UR6XGWZXHzLXfycmQKKYQ4+dflpiOJiEgMUxmRXjFyUBZlk79JwHYy8MTLBPY+bzqSiIjEKJUR6TW3ffRqnnB+FADf01/TdWtEROScVEak12S5Uuj/kW9RY+fSv/k9Wh6+Et7+M4RDpqOJiEgMURmRXnXDjNH8tv+9NNpu3LV74X/vJvDAFHjzNxBoMh1PRERigMqI9CrLsvjsHV/kX4f9if8IfoKTdg5p3nJ49iuEfjYBNv4UmutNxxQREYMs27Zt0yEuxOPxkJubS0NDAzk5OabjSDftrfDw67/tImffE3zO+X8UO2oACKdm4px+B8xeAjlFhlOKiEhP6ez3t8qI9LkDVV5+tWEf9q41fMH5DGMdRwGIOFJxTL4F5twDg0YbTikiIhdLZURi3qGaRh7acJD6t5/lC86nmOXYB4CNhVV6A1xxLwydbjiliIh0l8qIxI2jtU386uWDvLt9A59zPM21zm1tj9nDrsC64l4YeTVYlsGUIiLSVSojEneO1TfxyMZ32bZlM3daT7PI+RqpVhgAu2Ai1txlMG4ROFOM5hQRkc5RGZG4VdnQwq9feZcNb2znM/wftzo3kGn5AbD7D8Oa82WYchukphtOKiIi56MyInGv2tvCb/9+mKc37eYTkee4I+V58qxGAOzMQVizvggz7ob0fmaDiojIOamMSMKobfTzn68e5s+byrg+tJ7PpfwfQ62TANhp2VjT74DL/wlyCg0nFRGRs6mMSMI51RTgd68d4Y+vHWBe4FW+mPIMpY5yAGxnGtbkT0VPCx54qeGkIiICnf/+7vIMrK+88goLFy6kqKgIy7J48sknz7v9yy+/jGVZH1gqKyu7+tKS5PplpLH8mtFs/Ma1XDr/Tj7lvJ87Al/lzcgYrHAAtv8B+5fT4YnPwPFtF35CERGJCV0uIz6fj8mTJ/PQQw91ab+ysjIqKiralvz8/K6+tAgAOe5Uln54FK9+42pmLbiVL6X+gI/7v8268GVY2LD3afjNh+G/boR3N0DsD/6JiCS1Lp8jed1113Hdddd1+YXy8/Pp169fl/cT6UiWK4UvzhvJ4tnD+O83RvLNVybx08aDfCFlLTc5Xyfl8EY4vBEKJ8PcZTDuJnA4TccWEZH36bML5U2ZMoXCwkKuueYaXnvttb56WUkC6WlO7v6HEfz9ax/itoUf4T8y7mVey8/4fWgBzbig4i34yx3wy+mw9XcQbDEdWUREztLrZaSwsJBHHnmEv/71r/z1r3+luLiYq666iu3bt3e4j9/vx+PxtFtELsSd6mTxnGG8/NWrWHLzh/nP7C8yp+XnPBD6GKfIgrpDsPZeeGAivPr/oKXBdGQREeEiz6axLIs1a9awaNGiLu03b948SkpK+OMf/3jOx7/zne/w3e9+9wPrdTaNdEUwHGHNjuP86qWDVNXW8SnnS3w+9VkKqY1u4MqB6XfC5V+C7AKzYUVEElCvnU3TE2bOnMnBgwc7fHzFihU0NDS0LeXl5X2YThJFqtPBJ6cX87fl81h5y+W8kveP/EPL/2N54IsctIeC3wOvPRAdKXnmHqh913RkEZGkZOQiHzt37qSwsOMJqlwuFy6Xqw8TSSJLcTpYNHUICycX8dyuCn6xvj/XVF3Bhx07WJr6DFPD+2Hbo7D9DzD2RrhiGRRNNR1bRCRpdLmMNDY2thvVOHz4MDt37iQvL4+SkhJWrFjB8ePH+cMf/gDAAw88wPDhwxk/fjwtLS389re/ZcOGDbz44os99y5EOsHpsPjopCKun1DIi3uq+MWGftx8YhrTrX0sTXuGq9gBe56MLiOugivuheHzdLVgEZFe1uUysnXrVj70oQ+13V++fDkAixcv5tFHH6WiooKjR4+2PR4IBPiXf/kXjh8/TkZGBpMmTeJvf/tbu+cQ6UsOh8VHJhSwYPxgNuyr5sEN/fhseSljrKP8U+paPurYhPPQy3DoZcgthrELo0vxLJ0aLCLSCzQdvCQ927Z55cBJfrH+AFvfq2eoVcPnU57lltRXcEWaz2yYOQhKb4gWk2FXQkqaudAiInFA16YR6SLbttl0qJYH1x9g86E6XAS40vE2N7u38yFrG+lh75mNXbkw5iPRYjLyakjLMBdcRCRGqYyIXIQtR+p49LUjvFRWTVMgTAohLnfsZZFrGwucW8kO1Z3ZOCUdRs2PHvw66lpI72cst4hILFEZEekBLcEwfz9wkhd2V/K3vVWcagriIMJU6wAL07axMHULA0JVZ3ZwpMKIedERkzE3QNYgc+FFRAxTGRHpYcFwhDcP1/HC7kpe2F1JlccP2Iy33uP61C0scm1nSPC9MztYDiiZHS0mpR+FfsXGsouImKAyItKLIhGbt46d4vndlbywq5IjtU0AjLSO8xHnVj6WvoORwf3tdyqa2npmzo0wcJSB1CIifUtlRKSP2LbN/qpGnt9VyfO7K9lbEb2W0hBquNa5lY9n7GB8cDcWZ/2nNqj0zCnDBZM0l4mIJCSVERFDjtY28cLuaDHZfrQe24aBNDDfuY2Pp2/nsvDbOO3QmR36lURHS8YuhKEzwWHkKg0iIj1OZUQkBlR7WnhxTxUv7K5k07u1hCI2Ofj4kGMHN6dvZ669g9SI/8wOWYPPmsvkH8CZai68iMhFUhkRiTENTUHW74sWk437a2gJRnDjZ57jbW5ybeND1nbSI41ndnDnwpjrW+cy+TCkppsLLyLSDSojIjGsKRDilf01vLC7ir/trcLbEiKVELMdu1mYto2POLeRHa4/s0NqBoy65sxcJm79dyAisU9lRCROBEIRNh+q5fndlby4u4qTjX4cRJhm7eeG1C0sTNvGgFD1mR2cadEL+Y1dGB05yRxoLLuIyPmojIjEoXDEZsfR+rYzc47VNwM2E6zDXJ+ylUWurRSFjp3ZwXLAJXNb5zK5AXKHGssuIvJ+KiMicc62bfZUeHihtZjsr4oeTzLSOs51zi3c7N7OyNDB9jsVXRa9unB+afT04UGlmp5eRIxRGRFJMIdqGnlhdxXP767krfJTAAy1aljg2MLN6dsZH9rbfi6T07ILo6UkfywMGgODxkbLiju3b9+AiCQdlRGRBFbR0MyLu6Nn5rxxuI5wxGYQp5jnfIuJKceZ5KpguF1Ov2B1x0+SXdQ6gtJaUk6XFZUUEekhKiMiSaLOF+Bve6t4cXclrxw4SSAUaXssmyZGWccY5YgWlIlpFQwLHyUnWNPxE+YMOfMTz9llRWfwiEgXqYyIJKFAKMLhkz72VXrYX+WlrLKRsioP5XXN7bbLwcel1nHGOI4xNb2K8aknKAkfJTtwvpIytP2xKKdHUlzZvfyuRCReqYyISBufP8T+Km+7glJW2cjJRn+77XJoZJR1nFLncaZnVDM25TjFoffIDJzs+Mlzi1uPRTldUE6XlKxeflciEutURkTkgmob/ZRVedlf6aWsyktZpZf9VY00+kPttsuhkdHWMSakVjAts4pSx3GGBN8j47wlpaT1WJTSMwfNDlRJEUkmKiMi0i22bXP8VDNlrQVlf6WXfZVe3q1pJBhu/89FLo2Mso4x1V3JtIwqRjuOUeg/QnqgtuMXyC0583NP/tjo3/7DIL2/rl4skmBURkSkRwXDEY6c9LUrKPurvLxX18T7/xXph5dR1nFmZlVxmbuSS61jDPYfwe0/T0lJSYecotZlyDluD4GMAbqqsUgcURkRkT7RFAhxsLoxOpJy1s891V7/B7btj4dS5wkuz65miquSkZQzqOUIrvOVlLM506LzpnRUVnKKICsfHM4efpci0h0qIyJiVL0vED1gtrWcnC4q3pbQB7Z1ESDfqmeoo45L3R5GuBoodtZTaNUxIHySnFAN6f7ac0/q9n6WM1pYcod0PMqSVQDOlF541yJyNpUREYk5tm1T6WmJ/sTTWlD2V3upbGih1hf4wM89Z0slRD71FFh1FFp1DHc1MDz1FEOc9QymlrzwSbICJ3EQ6fhJTrMckDW449GVnKLopHApaT335kWSkMqIiMSVUDhCrS9AtcdPtbeFaq+/7XaN10+1109N6xIIn7twOAkzkAYKrToKrDqKrFqGOuu5JPUUQxx1DLJr6Rc6iZNw50Jl5p9ndGUwZA0Cdz8deCvSgc5+f2ucUkRiQorTweAcN4Nz3EDHU9Lbts2ppmBbOXl/can2DuKk189Brz96inIYCJzZ3yLCQDytIyy1bSMtRVYdxSmnKLTqGGifJNUOgq86ulTs7Di4IxUyB0HmwOjxKqdvZ7bezhrUuq51cab21EcmkjBURkQkrliWRf/MNPpnpjGm4PyzvzYFQq0lpf0Iy+nictTrZ6vXT52vta0ET+9p0x9v2wjL2SMtBdQxxFnPQBrIwgeRIHhPRJfOcPc7q7SctbSVlvzWMjMoOrutRl0kCaiMiEjCykhLYdjAFIYNzDzvdsFwhJON/nbFpdrjp6Z13S5vCxtaR2JCEbuttLgIkIeXgVYDA6wGBloeBtLAAMsTXYeHQVb0fh4eUqwItJyKLif3XzB/xOkikjEQKysfR1Y+1rlKy+likzFAZxFJ3FIZEZGkl+p0UJibTmFu+nm3i0RsTjUHqfa2UO8L0tAcxNMc/Xt6OdEcZF/LmfunHw+Fw/SjkQFWa0HBc1ZpaWCQ5WFA6/qBVgOZlh9H2I/Dexy8xy/4Hmws/Gn9CboHEE6Pjqw4sweRmlNAWu5gHNn50VGZtMzo4sqO/k3N0OiLGKcyIiLSSQ6HRV5mGnmZXTvLxrZtmoPhM6WlqX2BqWwJsf99pcbf5MXZfJLUllpyI6cYaHlaS0t0pOV0aYmOunhxWDbuQB3uQB14DnQ+GxYhZzrh1Ezs1EwsVxYOVxYp6Vk4TheWtKwzJSYt+6zbWdHp/c++n5YFqekqONIlKiMiIr3Msiwy0lLISEu54OjLubScXWRay0xVS7CtwHiamgk11mL5anA2RQuMO1BLZrCe3EgDA63okk0TmVYLGfjJpAWHZWNhkxpuIjXcBC014O2Rd9y+wLiy3ldoznH/XKXm9P3UjGjB0c9QCUtlREQkxrlTnbhTna1nGnWNPxRu+7morinIQV+AU01B6nwtNHq9+BobaPF58Td5CDV7CLV4IeAjgxYyW5cM64O3Myx/67pmMi0/GbSQZbW0vqoNAW906UlOF6RltJaT1oKSlhn9e3rdhR5vt6717+l9nGka0TFEZUREJIG5UpzkZzvJz+58kQlHbBqag9T5AtQ3Bahv/VvrC3LwrPvRx4PUN0ULjkWEdAIfKDDR0Zjo30zO3D49QtMvxU+uM0C2w09W67ZuuxlXuInUSMtZwfzQ7Ifm+l74pIjO3ttWaDpZYM71uCsL0vOiBxWn99dsv52gT0hERNpxduPYmFA4QkNztJjUN7UWGV+AurbyEl138Kz7Dc2tpyV98AoBZ7FxEyAdPxn4cVsBMvBH71t+3ATIoIV0K4C7dZv01m0yHQGyHH4yrCAZVuv+BEi3/LhtPy67BZftJ+V0ADvcOyM6rlzI6N9aTvIgI++s2/3PFJeMvDOPp3b957x4pjIiIiIXLcXpYECWiwFZrk7vEwpHONUcbFdWomUmWlhqfQGa/GH8oTCBcAR/MNL21xsKEwhF8LcugVCkw5l5L5idEOmthSfd8pPeWnDOLj5n1vtxtxab9Nbik/6+cpRNE/2tRnLx4bBs8DdEl/ojnc4USUmHjDysjDysc5aYs4rL6dtxPC+NyoiIiBiR4nQwMMvFwC4UmPOJROxoWQlFogXmdFlpKzHhtuLiD0UIhMP4g5Gz1rV/3H/WuvpQhMpQ9DnaFaNQuF1J8ofCRFovsuIgQi6N9Lca6UcjeZaX/pa37Xa/1sf6W176423bLtUK4wg1g+d4dOmksJVCMK0/YXc/Iun9sdIH4MgcQEr2AFKyBuJoN/rSetvdDxyOHvn8L4bKiIiIJASHw8LtiB7sC+am3W8JhvH5Q3hbQjS2/vW2BGn0n7nf0BLiuD9IY+s2npZQ9HZLENvvJdVfR1YkWlCiRcVLP6uR/m1FxntWkWkk3QrgtEM4/TXgr4GGzmWNYNHszKElJZfgjY9QMH5u7344HVAZERER6UGnz37qyk9W72fbNv5QpK28NLaE8LaWF29LiHf97YuOv9kHTXVYLXWktNSTGjiFO3iK9GADuUSLTB5e+rWWl/5WIzlWEw5sMsMNZIYb2OuPUNCDn0NXqIyIiIjEGMuy2krNxfyMdbrUnBmlCXK8JcQ+fwhfUzMBby1hXy12Uy3XXDKuB99B16iMiIiIJKizS82g7HOVmuF9nulczB+1IiIiIklNZURERESMUhkRERERo1RGRERExCiVERERETFKZURERESMUhkRERERo1RGRERExCiVERERETFKZURERESMUhkRERERo1RGRERExCiVERERETEqLq7aa9s2AB6Px3ASERER6azT39unv8c7EhdlxOv1AlBcXGw4iYiIiHSV1+slNze3w8ct+0J1JQZEIhFOnDhBdnY2lmX12PN6PB6Ki4spLy8nJyenx55X2tPn3Hf0WfcNfc59Q59z3+jNz9m2bbxeL0VFRTgcHR8ZEhcjIw6Hg6FDh/ba8+fk5Oh/6H1An3Pf0WfdN/Q59w19zn2jtz7n842InKYDWEVERMQolRERERExKqnLiMvl4tvf/jYul8t0lISmz7nv6LPuG/qc+4Y+574RC59zXBzAKiIiIokrqUdGRERExDyVERERETFKZURERESMUhkRERERo5K6jDz00EMMGzYMt9vNrFmzePPNN01HSigrV65kxowZZGdnk5+fz6JFiygrKzMdK+H96Ec/wrIsli1bZjpKwjl+/Dif/vSnGTBgAOnp6UycOJGtW7eajpVwwuEw9913H8OHDyc9PZ2RI0fyve9974LXN5Hze+WVV1i4cCFFRUVYlsWTTz7Z7nHbtvm3f/s3CgsLSU9PZ/78+Rw4cKBPsiVtGXniiSdYvnw53/72t9m+fTuTJ09mwYIFVFdXm46WMDZu3MiSJUvYvHkz69atIxgMcu211+Lz+UxHS1hbtmzh17/+NZMmTTIdJeHU19czd+5cUlNTee6559izZw/3338//fv3Nx0t4fz4xz/m4Ycf5pe//CV79+7lxz/+MT/5yU/4xS9+YTpaXPP5fEyePJmHHnronI//5Cc/4cEHH+SRRx7hjTfeIDMzkwULFtDS0tL74ewkNXPmTHvJkiVt98PhsF1UVGSvXLnSYKrEVl1dbQP2xo0bTUdJSF6v1x41apS9bt06e968efY999xjOlJC+frXv25fccUVpmMkhRtuuMG+884726372Mc+Zt92222GEiUewF6zZk3b/UgkYhcUFNg//elP29adOnXKdrlc9p/+9Kdez5OUIyOBQIBt27Yxf/78tnUOh4P58+ezadMmg8kSW0NDAwB5eXmGkySmJUuWcMMNN7T737X0nKeffprp06fziU98gvz8fKZOncpvfvMb07ES0pw5c1i/fj379+8H4K233uLVV1/luuuuM5wscR0+fJjKysp2/37k5uYya9asPvlejIsL5fW0kydPEg6HGTx4cLv1gwcPZt++fYZSJbZIJMKyZcuYO3cuEyZMMB0n4Tz++ONs376dLVu2mI6SsA4dOsTDDz/M8uXL+eY3v8mWLVv453/+Z9LS0li8eLHpeAnlG9/4Bh6Ph9LSUpxOJ+FwmB/84AfcdtttpqMlrMrKSoBzfi+efqw3JWUZkb63ZMkSdu3axauvvmo6SsIpLy/nnnvuYd26dbjdbtNxElYkEmH69On88Ic/BGDq1Kns2rWLRx55RGWkh/3P//wP//3f/83q1asZP348O3fuZNmyZRQVFemzTlBJ+TPNwIEDcTqdVFVVtVtfVVVFQUGBoVSJa+nSpaxdu5aXXnqJoUOHmo6TcLZt20Z1dTWXXXYZKSkppKSksHHjRh588EFSUlIIh8OmIyaEwsJCxo0b127d2LFjOXr0qKFEieurX/0q3/jGN/jUpz7FxIkT+cxnPsO9997LypUrTUdLWKe/+0x9LyZlGUlLS2PatGmsX7++bV0kEmH9+vXMnj3bYLLEYts2S5cuZc2aNWzYsIHhw4ebjpSQrr76at555x127tzZtkyfPp3bbruNnTt34nQ6TUdMCHPnzv3Aqen79+/nkksuMZQocTU1NeFwtP96cjqdRCIRQ4kS3/DhwykoKGj3vejxeHjjjTf65HsxaX+mWb58OYsXL2b69OnMnDmTBx54AJ/Pxx133GE6WsJYsmQJq1ev5qmnniI7O7vtd8fc3FzS09MNp0sc2dnZHzgOJzMzkwEDBuj4nB507733MmfOHH74wx/yyU9+kjfffJNVq1axatUq09ESzsKFC/nBD35ASUkJ48ePZ8eOHfzsZz/jzjvvNB0trjU2NnLw4MG2+4cPH2bnzp3k5eVRUlLCsmXL+P73v8+oUaMYPnw49913H0VFRSxatKj3w/X6+Tox7Be/+IVdUlJip6Wl2TNnzrQ3b95sOlJCAc65/P73vzcdLeHp1N7e8cwzz9gTJkywXS6XXVpaaq9atcp0pITk8Xjse+65xy4pKbHdbrc9YsQI+1vf+pbt9/tNR4trL7300jn/TV68eLFt29HTe++77z578ODBtsvlsq+++mq7rKysT7JZtq0p7URERMScpDxmRERERGKHyoiIiIgYpTIiIiIiRqmMiIiIiFEqIyIiImKUyoiIiIgYpTIiIiIiRqmMiIiIiFEqIyIiImKUyoiIiIgYpTIiIiIiRqmMiIiIiFH/H9+T+HZ8SA9XAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "plt.plot(train_losses)\n",
    "plt.plot(val_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, \"gpt2_14m_clean.pth\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}