part1_torch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32033 15 2\n"
     ]
    }
   ],
   "source": [
    "with open(\"./names.txt\") as f:\n",
    "    words = f.read().splitlines()\n",
    "    print(len(words), max(len(w) for w in words), min(len(w) for w in words))\n",
    "chars = sorted(list(set((''.join(words)))))  #get unique characters \n",
    "stoi = {s:i+1 for i, s in enumerate(chars)}  # map char to int\n",
    "stoi[\".\"] = 0 \n",
    "itos = {i:s for s,i in stoi.items()}         # map int to char\n",
    "vocab_size = len(itos)\n",
    "#create training set \n",
    "xs, ys = [], []\n",
    "n = 0\n",
    "for w in words:\n",
    "    chs = ['.'] + list(w) + ['.']\n",
    "    for ch1, ch2 in zip(chs, chs[1:]):\n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        # print(ch1,ch2) be carful with prints and for loops long for loops are very hard to stop ,kernel become un alive \n",
    "        xs.append(ix1)\n",
    "        ys.append(ix2)\n",
    "\n",
    "xs = F.one_hot(torch.tensor(xs)).float()\n",
    "ys = torch.tensor(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BigramModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.W = nn.Linear(vocab_size, vocab_size, bias=False)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        return self.W(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BigramModel()\n",
    "optimizer = optim.SGD(model.parameters(), lr=1, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.3106322288513184\n",
      "3.305067300796509\n",
      "3.294562816619873\n",
      "3.2797439098358154\n",
      "3.261221408843994\n",
      "3.2395942211151123\n",
      "3.2154417037963867\n",
      "3.1893250942230225\n",
      "3.161783218383789\n",
      "3.133331537246704\n",
      "3.104456663131714\n",
      "3.0756115913391113\n",
      "3.047208309173584\n",
      "3.0196053981781006\n",
      "2.9931013584136963\n",
      "2.9679269790649414\n",
      "2.9442331790924072\n",
      "2.9220945835113525\n",
      "2.901512861251831\n",
      "2.8824257850646973\n",
      "2.8647236824035645\n",
      "2.848266363143921\n",
      "2.8329017162323\n",
      "2.818483352661133\n",
      "2.804877996444702\n",
      "2.7919762134552\n",
      "2.7796945571899414\n",
      "2.7679738998413086\n",
      "2.756779193878174\n",
      "2.74609112739563\n",
      "2.7359046936035156\n",
      "2.7262203693389893\n",
      "2.7170419692993164\n",
      "2.7083733081817627\n",
      "2.7002127170562744\n",
      "2.692552089691162\n",
      "2.685377359390259\n",
      "2.6786680221557617\n",
      "2.672395706176758\n",
      "2.6665289402008057\n",
      "2.6610329151153564\n",
      "2.6558711528778076\n",
      "2.6510088443756104\n",
      "2.646411180496216\n",
      "2.6420490741729736\n",
      "2.637894630432129\n",
      "2.633925676345825\n",
      "2.6301229000091553\n",
      "2.6264705657958984\n",
      "2.622957706451416\n",
      "2.619575023651123\n",
      "2.6163156032562256\n",
      "2.6131739616394043\n",
      "2.6101465225219727\n",
      "2.607229232788086\n",
      "2.6044199466705322\n",
      "2.601714849472046\n",
      "2.5991106033325195\n",
      "2.5966031551361084\n",
      "2.594189405441284\n",
      "2.5918655395507812\n",
      "2.589625835418701\n",
      "2.5874667167663574\n",
      "2.585383176803589\n",
      "2.5833704471588135\n",
      "2.5814244747161865\n",
      "2.5795414447784424\n",
      "2.577716588973999\n",
      "2.5759472846984863\n",
      "2.5742297172546387\n",
      "2.572561025619507\n",
      "2.570939064025879\n",
      "2.5693609714508057\n",
      "2.5678253173828125\n",
      "2.5663299560546875\n",
      "2.5648739337921143\n",
      "2.563455820083618\n",
      "2.5620734691619873\n",
      "2.5607264041900635\n",
      "2.5594139099121094\n",
      "2.558133602142334\n",
      "2.5568854808807373\n",
      "2.5556681156158447\n",
      "2.5544800758361816\n",
      "2.553321123123169\n",
      "2.5521891117095947\n",
      "2.55108380317688\n",
      "2.550004005432129\n",
      "2.548948287963867\n",
      "2.5479161739349365\n",
      "2.5469067096710205\n",
      "2.5459187030792236\n",
      "2.544952392578125\n",
      "2.5440056324005127\n",
      "2.5430784225463867\n",
      "2.5421700477600098\n",
      "2.5412797927856445\n",
      "2.540407419204712\n",
      "2.5395522117614746\n",
      "2.5387139320373535\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):    \n",
    "    logits = model(xs)\n",
    "    loss = F.cross_entropy(logits, ys)\n",
    "    print(loss.item())    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "on.\n",
      "ctryshayan.\n",
      "ataleilemfft.\n",
      "laymbrineriromfimzain.\n",
      "rewnalliole.\n",
      "enyhke.\n",
      "mkangg.\n",
      "enannfayn.\n",
      "qnelynn.\n",
      "tovlirmigefbuwthan.\n",
      "e.\n",
      "ete.\n",
      "yn.\n",
      "ttyapwan.\n",
      "al.\n",
      "ja.\n",
      "l.\n",
      "curennisxiovetsam.\n",
      "a.\n",
      "lebrnireyahh.\n"
     ]
    }
   ],
   "source": [
    "#Sample from the model\n",
    "for i in range(20):\n",
    "\n",
    "  ix = 0\n",
    "  out = []\n",
    "  \n",
    "  while True:\n",
    "    # p=P[ix]   \n",
    "    \n",
    "    logits = model(F.one_hot(torch.tensor([ix]),num_classes=27).float())\n",
    "    counts = logits.exp()\n",
    "    p = counts/counts.sum(1,keepdims=True)\n",
    "    ix = torch.multinomial(p,num_samples=1,replacement=True).item()\n",
    "    \n",
    "    out.append(itos[ix])\n",
    "    if ix == 0:\n",
    "      break\n",
    "  print(''.join(out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YAYYY"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}