transfomers-explained-mathematic-and-code-in-depth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Chrome executable not able to be found on your machine",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtransfomers-explained-mathematic-and-code-in-depth.ipynb\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      3\u001b[0m Token \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2e7f149b53d30d3fecf75a5f4805d727b9633eb754b09e414dfb780a6e4938596\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 4\u001b[0m \u001b[43mjupyter_to_medium\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpublish\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mToken\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/jupyter_to_medium/_publish_to_medium.py:432\u001b[0m, in \u001b[0;36mpublish\u001b[0;34m(filename, integration_token, pub_name, title, tags, publish_status, notify_followers, license, canonical_url, chrome_path, save_markdown, table_conversion, gistify, gist_threshold, public_gists)\u001b[0m\n\u001b[1;32m    331\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpublish\u001b[39m(\n\u001b[1;32m    332\u001b[0m     filename,\n\u001b[1;32m    333\u001b[0m     integration_token\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    346\u001b[0m     public_gists\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    347\u001b[0m ):\n\u001b[1;32m    348\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    349\u001b[0m \u001b[38;5;124;03m    Publish a Jupyter Notebook directly to Medium as a blog post.\u001b[39;00m\n\u001b[1;32m    350\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    430\u001b[0m \u001b[38;5;124;03m        or private (only accessible through link).\u001b[39;00m\n\u001b[1;32m    431\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 432\u001b[0m     p \u001b[38;5;241m=\u001b[39m \u001b[43mPublish\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    433\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    434\u001b[0m \u001b[43m        \u001b[49m\u001b[43mintegration_token\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    435\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpub_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    436\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtitle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    437\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    438\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpublish_status\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    439\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnotify_followers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    440\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlicense\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    441\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcanonical_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    442\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchrome_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    443\u001b[0m \u001b[43m        \u001b[49m\u001b[43msave_markdown\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    444\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtable_conversion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    445\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgistify\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    446\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgist_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    447\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpublic_gists\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    448\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    449\u001b[0m     p\u001b[38;5;241m.\u001b[39mmain()\n\u001b[1;32m    450\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m p\u001b[38;5;241m.\u001b[39mresult\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/jupyter_to_medium/_publish_to_medium.py:59\u001b[0m, in \u001b[0;36mPublish.__init__\u001b[0;34m(self, filename, integration_token, pub_name, title, tags, publish_status, notify_followers, license, canonical_url, chrome_path, save_markdown, table_conversion, gistify, gist_threshold, public_gists)\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpublic_gists \u001b[38;5;241m=\u001b[39m public_gists\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb_home \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfilename\u001b[38;5;241m.\u001b[39mparent\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresources \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_resources\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     60\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_notebook()\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_headers()\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/jupyter_to_medium/_publish_to_medium.py:100\u001b[0m, in \u001b[0;36mPublish.get_resources\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtable_conversion \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchrome\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_screenshot\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Screenshot\n\u001b[0;32m--> 100\u001b[0m     converter \u001b[38;5;241m=\u001b[39m \u001b[43mScreenshot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcenter_df\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfontsize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m14\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchrome_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchrome_path\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mrun\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    102\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_matplotlib_table\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TableMaker\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/jupyter_to_medium/_screenshot.py:92\u001b[0m, in \u001b[0;36mScreenshot.__init__\u001b[0;34m(self, center_df, max_rows, max_cols, chrome_path, fontsize, encode_base64, limit_crop)\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mss_width \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1200\u001b[39m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mss_height \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m900\u001b[39m\n\u001b[0;32m---> 92\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchrome_path \u001b[38;5;241m=\u001b[39m \u001b[43mget_chrome_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchrome_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_css(fontsize)\n\u001b[1;32m     94\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencode_base64 \u001b[38;5;241m=\u001b[39m encode_base64\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/jupyter_to_medium/_screenshot.py:60\u001b[0m, in \u001b[0;36mget_chrome_path\u001b[0;34m(chrome_path)\u001b[0m\n\u001b[1;32m     58\u001b[0m             \u001b[38;5;28;01mif\u001b[39;00m chrome_path:\n\u001b[1;32m     59\u001b[0m                 \u001b[38;5;28;01mreturn\u001b[39;00m chrome_path\n\u001b[0;32m---> 60\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mChrome executable not able to be found on your machine\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m system \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwindows\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m     62\u001b[0m     \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwinreg\u001b[39;00m\n",
      "\u001b[0;31mOSError\u001b[0m: Chrome executable not able to be found on your machine"
     ]
    }
   ],
   "source": [
    "import jupyter_to_medium\n",
    "filename = \"transfomers-explained-mathematic-and-code-in-depth.ipynb\"\n",
    "Token = \"2e7f149b53d30d3fecf75a5f4805d727b9633eb754b09e414dfb780a6e4938596\"\n",
    "jupyter_to_medium.publish(filename, Token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Self-Attention</h2>\n",
    "            <p>Self-Attention is a mechanism used in neural networks, particularly in the Transformer model, to capture the importance of different words in a sentence or sequence. The mathematical formula for Self-Attention is as follows:</p>\n",
    "            <center><img src=\"https://latex.codecogs.com/svg.image?\\text{Attention}(Q,K,V)&space;=&space;\\text{softmax}\\left(\\frac{QK^T}{\\sqrt{d_k}}\\right)V\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\"/>\n",
    "</center>\n",
    "            <p>Where:</p>\n",
    "            <ul>\n",
    "                <li>$Q$ is the Query matrix</li>\n",
    "                <li>$K$ is the Key matrix</li>\n",
    "                <li>$V$ is the Value matrix</li>\n",
    "                <li>$d_k$ is the dimension of $K$ (or $Q$)</li>\n",
    "                <strong>Note:</strong><p>the Matrcies are learnble wieghts</p>\n",
    "            </ul>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Re-implmentation of Paper <strong>Attention All you need</strong> </h2>\n",
    "            <p>In this section : we will code tow versions of self-Attention Numpy version and Torch Built Self-Attention. </p>\n",
    "            <p>Recently all the most State-of-The-Art model in NLP is based on <strong>Self-Attention Mechanism</strong> because is Powerfull neural network Architucture to learn in parallel and extarct important feartures from data that why Self-Attention is Data hungry is require hugh amount of data to learn from even it's comes with own limitation we will mention at the end of Notebook</p>\n",
    "            <div class=\"alert alert-warning\">\n",
    "                <p><strong>Reading:</strong> For more information about Self-Attention , please refer to the following paper: </p>\n",
    "                <ul>\n",
    "                    <li><a href=\"https://arxiv.org/abs/1706.03762\">Attention All you need Sequentail modeling Seq2Seq</a></li>\n",
    "                </ul>\n",
    "            </div>\n",
    "        </div>\n",
    "            </div>\n",
    "        </div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-27T15:36:01.985208Z",
     "iopub.status.busy": "2023-05-27T15:36:01.984753Z",
     "iopub.status.idle": "2023-05-27T15:36:03.279154Z",
     "shell.execute_reply": "2023-05-27T15:36:03.278137Z",
     "shell.execute_reply.started": "2023-05-27T15:36:01.985165Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np \n",
    "import torch.nn as nn \n",
    "import matplotlib.pyplot as plt \n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Self-Attention componentes</h2></br>\n",
    "            <strong>Notation : all the mentioned ideas here comes from this book called DL2 </strong>\n",
    "            <div class=\"alert alert-success\">\n",
    "                <p><strong>Reading:</strong> For more information about Self-Attention and mathematic , please refer to the following Book: </p>\n",
    "<ul>\n",
    "    <li><a href=\"https://d2l.ai/index.html2\">Dive into Deep Learning</a></li>\n",
    "                </ul>\n",
    "    </div>\n",
    "       <p>Self-Attention Mechanism (SAM) creete to imporve sequenctial modeling performence that solve recuccrent and long memory which was most limittaion comes over LTSM and RNN models and one of most archive by SAM is Parallel process of sequences we dived the this process by Query and Key and Value easise way to ullistat how does SAM works by taking for example Database search called <strong>look-up Table</strong>\n",
    "                <ul>\n",
    "                    <li>Query : is the SLQ commmand that goes into Database system to find any similar content</li>\n",
    "                    <li>Key :paly the role of what're you looking for such for Key==\"Deep learning Course\"</li>\n",
    "                    <li>Value : we can say is target content we want to have</li></ul><p> may ask yourself since that Query and Key and Value a have different means but the feed the same input of data ,here where's SAM comes to play around and find similarty between Q and K , V now let us first defined SAM formula in depth </p>\n",
    "       </div>\n",
    "    </div>\n",
    "    </div>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Explain mechanism of Attention in Encoder Part</h2>\n",
    "            <ul><li>\n",
    "                <strong>Dot Product of Query Transpose and Key</strong><center>\n",
    "                Let us consider a sequence of $n$ embeddings $\\mathbf{E} = [\\mathbf{e_1}, \\mathbf{e_2}, \\ldots, \\mathbf{e_n}]$. \n",
    "                \n",
    "##### <strong>Step 1: intialize the learnbel wieghts matrices</br> Query, Key, Value</strong>\n",
    "                \n",
    "We create three linear projections of each embedding $\\mathbf{e_i}$ using three learned matrices $\\mathbf{W_Q}$, $\\mathbf{W_K}$, $\\mathbf{W_V}$:\n",
    "\\begin{align*}\n",
    "\\mathbf{Q_i} &= \\mathbf{W_Q} \\mathbf{e_i} \\\\\n",
    "\\mathbf{K_i} &= \\mathbf{W_K} \\mathbf{e_i} \\\\\n",
    "\\mathbf{V_i} &= \\mathbf{W_V} \\mathbf{e_i}\n",
    "\\end{align*}\n",
    "                \n",
    "These projections are called the **Query**, **Key**, and **Value** matrices, respectively.\n",
    "                <center><img src=\"https://wikidocs.net/images/page/178172/Fig_08_03.png\" width=\"500\" height=\"500\"/></center>\n",
    "        \n",
    "##### <strong>Step 2: Dot-Product Similarity</strong>\n",
    "\n",
    "We then compute the dot-product similarity between the Query ($\\mathbf{Q}$) and Key ($\\mathbf{K}$) matrices:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\text{Similarity}(\\mathbf{Q}, \\mathbf{K}) = \\mathbf{Q} \\mathbf{K}^T\n",
    "\\end{equation*}\n",
    "\n",
    "This results in a square matrix of size $n \\times n$.</center>\n",
    "            <strong>Summary First Step</strong><p>the reason behind doing the dot product is to calculate the similarity between to Vectors in Eucldien Space which means tell us how much two vector are close to each but the output from Dot product not Scale that's why we need to averge the out weights matrix between 1 and 0</p>\n",
    "            </li>\n",
    "        <li>\n",
    "           <strong>Attention Scoring Functions</strong><center>       \n",
    "\\begin{equation*}\n",
    "\\mathbf{S} = \\text{softmax}(\\text{Similarity}(\\mathbf{Q}, \\mathbf{K})) = \\text{softmax}(\\mathbf{Q} \\mathbf{K}^T)\n",
    "\\end{equation*}\n",
    "\n",
    "The softmax function ensures that the weights for each Value are positive, and sum to 1.\n",
    "            <p><center><p>The dot product between both vectors has zero mean and a variance of . To ensure that the variance of the dot product still remains one regardless of vector length, we use the scaled dot-product attention scoring function. That is, we rescale the dot-product by . We thus arrive at the first commonly used attention function that is used, e.g. in Transformers</p><img src=\"https://d2l.ai/_images/attention-output.svg\" width=\"500\" height=\"500\"/></center></br>\n",
    "        <text>Fig.1: Computing the output of attention pooling as a weighted average of values, where weights are computed with the attention scoring function and the softmax operation.</text></p>\n",
    "          </center></li>\n",
    "          <li><strong>Final stage</strong><center>\n",
    "##### <strong>Weighted Sum</strong>\n",
    "\n",
    "We compute the weighted sum of the Value matrix using the Softmax weights:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\text{Weighted Sum}(\\mathbf{V}, \\mathbf{S}) = \\mathbf{S} \\mathbf{V}\n",
    "\\end{equation*}\n",
    "\n",
    "This results in a matrix of size $n \\times d$, where $d$ is the number of features in each Value vector. \n",
    "\n",
    "##### <strong>Output</strong>\n",
    "\n",
    "We obtain the final output matrix by concatenating the Weighted Sum matrices computed for each embedding in the input sequence:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\text{Output} = [\\text{Weighted Sum}(\\mathbf{V_1}, \\mathbf{S}), \\text{Weighted Sum}(\\mathbf{V_2}, \\mathbf{S}), \\ldots, \\text{Weighted Sum}(\\mathbf{V_n}, \\mathbf{S})]\n",
    "\\end{equation*}\n",
    "\n",
    "The Self-Attention mechanism can be summarized with the following formula:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\text{Self-Attention}(\\mathbf{Q}, \\mathbf{K}, \\mathbf{V}) = \\text{softmax}\\left(\\frac{\\mathbf{Q} \\mathbf{K}^T}{\\sqrt{d_k}}\\right) \\mathbf{V}\n",
    "\\end{equation*}\n",
    "Where:</br>\n",
    "$\\mathbf{Q}$ is the Query weighted matrix</br>\n",
    "$\\mathbf{K}$ is the Key weighted  matrix</br>\n",
    "$\\mathbf{V}$ is the Value weighted matrix</br> following\n",
    "$d_k$ is the dimension of $\\mathbf{K}$ (or $\\mathbf{Q}$)\n",
    "    </center>\n",
    "</li></ul>\n",
    "</div>\n",
    "     </div>\n",
    "        </div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": false,
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:08.738526Z",
     "iopub.status.busy": "2023-05-11T13:48:08.738178Z",
     "iopub.status.idle": "2023-05-11T13:48:08.753811Z",
     "shell.execute_reply": "2023-05-11T13:48:08.752977Z",
     "shell.execute_reply.started": "2023-05-11T13:48:08.738494Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def Softmax(z):\n",
    "    \"\"\"\n",
    "    descriptions function : Softmax is non-linear function that give the averege of between \n",
    "    0 and 1 of in element in matrix \n",
    "    \"\"\"\n",
    "    e_x = np.exp(z - z.max(axis=-1,keepdims=True))\n",
    "    return e_x / np.sum(e_x , axis=-1 ,keepdims=True)\n",
    "\n",
    "def Self_Attention(input_embedding ,WieghtMatrix_QKY, out_wieghts,mask=None,batch_first=True) :\n",
    "    \"\"\"\n",
    "    Self-Attention take input of emebeding matrix which asseccoite with \n",
    "    the Positional encoding ww will cover later in section \n",
    "    Query and Key and Value all of them have the same dimession as the input \n",
    "    \"\"\"\n",
    "    try : \n",
    "        if batch_first==True:\n",
    "            Query , Key , value = np.split(input_embedding@WieghtMatrix_QKY , 3 , axis=-1)\n",
    "            if mask is not None:\n",
    "                assert mask.shape[0] == input_embedding.shape[1],\\\n",
    "                    f\"input dimession of mask doesn't match with dimession of embedding input:{mask.shape[0]} {input_embedding.shape[0]}\"\n",
    "                Attention = Softmax(Key@Query.swapaxes(-1,-2) / np.sqrt(input_embedding.shape[-1]) + mask) \n",
    "                return  Attention@value@out_wieghts , Attention\n",
    "            else:\n",
    "                Attention = Softmax(Key@Query.swapaxes(-1,-2) / np.sqrt(input_embedding.shape[-1])) \n",
    "                return  Attention@value@out_wieghts , Attention\n",
    "    except:\n",
    "        raise Exception(\"Batch argumment is missing\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:08.756501Z",
     "iopub.status.busy": "2023-05-11T13:48:08.755650Z",
     "iopub.status.idle": "2023-05-11T13:48:08.884226Z",
     "shell.execute_reply": "2023-05-11T13:48:08.883260Z",
     "shell.execute_reply.started": "2023-05-11T13:48:08.756450Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "now let us compute Self-Attention using \n",
    "Torch Built Multi-Heads Self-Attention\n",
    "the reason we would to compare \n",
    "the out_wieghts matrix with out \n",
    "similar Attention out_projt_weight to use \n",
    "\"\"\"\n",
    "## intialize the input \n",
    "batch,lenght_sequnece , embedding_size = 1,100 , 64 \n",
    "number_heads = 1 \n",
    "# now we will stand out with single Attetion heads \n",
    "input_embedding = torch.randn(batch,lenght_sequnece, embedding_size)\n",
    "atten = nn.MultiheadAttention(embedding_size,1, bias = False,batch_first=True)\n",
    "## intialzie the Mask \n",
    "Mask = torch.triu(-float(\"inf\")*torch.ones(lenght_sequnece,lenght_sequnece),1)       \n",
    "out_weight_average , attention = atten(input_embedding,input_embedding,input_embedding,attn_mask=Mask)                  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:08.886444Z",
     "iopub.status.busy": "2023-05-11T13:48:08.885763Z",
     "iopub.status.idle": "2023-05-11T13:48:08.892078Z",
     "shell.execute_reply": "2023-05-11T13:48:08.890951Z",
     "shell.execute_reply.started": "2023-05-11T13:48:08.886404Z"
    }
   },
   "outputs": [],
   "source": [
    "### intialize the in_projecion weights and out_projetction weights\n",
    "print(f\"the in_projecion weights: {atten.in_proj_weight.shape}\\nThe out_projetction weights: {atten.out_proj.weight.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:08.896638Z",
     "iopub.status.busy": "2023-05-11T13:48:08.895736Z",
     "iopub.status.idle": "2023-05-11T13:48:08.907740Z",
     "shell.execute_reply": "2023-05-11T13:48:08.906494Z",
     "shell.execute_reply.started": "2023-05-11T13:48:08.896600Z"
    }
   },
   "outputs": [],
   "source": [
    "WieghtMatrix_QKV = atten.in_proj_weight\n",
    "out_Wieghts = atten.out_proj.weight\n",
    "out_wieghts_Averege , atten = Self_Attention(input_embedding.numpy(),\n",
    "                                     WieghtMatrix_QKV.detach().numpy().T,\n",
    "                                     out_Wieghts.detach().numpy().T,  \n",
    "                                     mask=None,batch_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Simulation of Self-Attention Learning at Stage Encoder Part</h2>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_08_02_encoder_self_attention.gif\" width=\"500\" height=\"500\"/></center>\n",
    "    <div style=\"font-size: 14px;\">This image shows a simulation of the Self-Attention mechanism at the Encoder stage of a Transformer model. Each of the input embeddings (represented by the colored squares) attends to all the other embeddings, and produces a weighted sum of the Value embeddings (represented by the circles). This process is repeated for each of the Encoder layers, and the final output is passed on to the Decoder stage.</div>\n",
    "</div>\n",
    "   </div>\n",
    "    </div>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:08.910793Z",
     "iopub.status.busy": "2023-05-11T13:48:08.909906Z",
     "iopub.status.idle": "2023-05-11T13:48:09.448375Z",
     "shell.execute_reply": "2023-05-11T13:48:09.447106Z",
     "shell.execute_reply.started": "2023-05-11T13:48:08.910742Z"
    }
   },
   "outputs": [],
   "source": [
    "fig , axe = plt.subplots(1 , 1 , figsize=(6,6))\n",
    "axe.set_title(\"the Matrix Similarty of Attention Score function \")\n",
    "axe.set_xlabel(\"Key\")\n",
    "axe.set_ylabel(\"Query\")\n",
    "im =axe.imshow(atten[0][:20,:20], cmap=\"hot\",interpolation=\"nearest\")\n",
    "divider = make_axes_locatable(axe)\n",
    "cax = divider.append_axes(\"right\", size=\"4%\", pad=0.2)\n",
    "cbar = plt.colorbar(im, cax=cax)\n",
    "fig.canvas.draw()\n",
    "fig.suptitle(\"Similarty Matrix Across Dot Product Encoder\")\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Explain mechanism of Mask-Self-Attention in Decoder Part</h2>\n",
    "            <p>here only diffrent between prevouis Self-Attention is that Self-attention in decoders is slightly different than in encoders. The encoder receives all the tokens at once and the tokens can see all the tokens in the input sentence, but the decoder generates one token at a time. During creation, you don't know which tokens you will create in the future.\n",
    "\n",
    "To prevent the decoder from looking ahead, the model uses masked self-attention. Future tokens are masked.</p>\n",
    "          <li><strong>Final stage</strong><center>\n",
    " ##### Softmax with Mask\n",
    "              \n",
    "We apply a softmax function to the Similarity matrix after adding the mask matrix:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\mathbf{S} = \\text{softmax}(\\text{Similarity}(\\mathbf{Q}, \\mathbf{K}) + \\mathbf{M}) = \\text{softmax}\\left(\\frac{\\mathbf{Q} \\mathbf{K}^T}{\\sqrt{d_k}} + \\mathbf{M}\\right)\n",
    "\\end{equation*}\n",
    "\n",
    "Here, $\\mathbf{M}$ is a mask matrix that prevents the decoder from \"looking ahead\" in the sequence during training. It has the same dimensions as the Similarity matrix and is typically a lower-triangular matrix.\n",
    "\n",
    "This modified Self-Attention mechanism formula can be summarized as follows:\n",
    "\n",
    "\\begin{equation*}\n",
    "\\text{Self-Attention}(\\mathbf{Q}, \\mathbf{K}, \\mathbf{V}, \\mathbf{M}) = \\text{softmax}\\left(\\frac{\\mathbf{Q} \\mathbf{K}^T}{\\sqrt{d_k}} + \\mathbf{M}\\right) \\mathbf{V}\n",
    "\\end{equation*}\n",
    "\n",
    "where $\\mathbf{Q}$, $\\mathbf{K}$, and $\\mathbf{V}$ are the Query, Key, and Value matrices, respectively, and $d_k$ is the dimensionality of the Key matrix.</br>\n",
    "   </center></li></ul>\n",
    "  </div>\n",
    "    </div>\n",
    "    </div>\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.451025Z",
     "iopub.status.busy": "2023-05-11T13:48:09.449859Z",
     "iopub.status.idle": "2023-05-11T13:48:09.461015Z",
     "shell.execute_reply": "2023-05-11T13:48:09.459739Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.450975Z"
    }
   },
   "outputs": [],
   "source": [
    "out_wieghts_Averege , atten = Self_Attention(input_embedding.numpy(),\n",
    "                                     WieghtMatrix_QKV.detach().numpy().T,\n",
    "                                     out_Wieghts.detach().numpy().T,  \n",
    "                                     mask=Mask.numpy(),batch_first=True)\n",
    "                                     \n",
    "                                   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Simulation of Masked-Attention Learning at Stage Decoder Part</h2>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_08_05_masked_self_attn.gif\" width=\"400\" height=\"200\"/></center>\n",
    "    <div style=\"font-size: 14px;\"> A simulation of Masked Self-Attention mechanism used in the Decoder stage of a Transformer model. The mechanism prevents the model from \"looking ahead\" in the sequence during training, ensuring that it only generates output tokens based on previously generated tokens..</div>\n",
    "</div>\n",
    "  </div>\n",
    "    </div>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.463980Z",
     "iopub.status.busy": "2023-05-11T13:48:09.463098Z",
     "iopub.status.idle": "2023-05-11T13:48:09.837172Z",
     "shell.execute_reply": "2023-05-11T13:48:09.835837Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.463911Z"
    }
   },
   "outputs": [],
   "source": [
    "fig , axe = plt.subplots(1 , 1 , figsize=(6,6))\n",
    "axe.set_title(\"the Matrix Similarty of Attention Score function with Mask \")\n",
    "axe.set_xlabel(\"Key\")\n",
    "axe.set_ylabel(\"Query\")\n",
    "im =axe.imshow(atten[0][:20,:20], cmap=\"hot\",interpolation=\"nearest\")\n",
    "divider = make_axes_locatable(axe)\n",
    "cax = divider.append_axes(\"right\", size=\"4%\", pad=0.2)\n",
    "cbar = plt.colorbar(im, cax=cax)\n",
    "fig.canvas.draw()\n",
    "fig.suptitle(\"Similarty Matrix Across Dot Product Decoder\")\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Multi-Heads Self-Attention</h2>\n",
    "            <p>In the Transformer, the Attention module repeats its computations multiple times in parallel. Each of these is called an Attention Head. The Attention module splits its Query, Key, and Value parameters N-ways and passes each split independently through a separate Head. All of these similar Attention calculations are then combined together to produce a final Attention score. This is called Multi-head attention and gives the Transformer greater power to encode multiple relationships and nuances for each word.:</p>\n",
    "            <center><img src=\"https://data-science-blog.com/wp-content/uploads/2022/01/mha_img_original.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"200\"/>\n",
    "</center>\n",
    "            <center>\n",
    "                <p>So we need to make the model focus on something else. This is the motivation behind Multi-Head Attention. Instead of using one attention mechanism, multi-head attention has multiple “heads” that work independently</p>\n",
    "                <img src=\"https://miro.medium.com/v2/resize:fit:786/0*X0c962yMhgRKfMTD.gif\" width=\"400\" height=\"200\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\"/>\n",
    "            </center>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Multi-Heads Self-Attention mathematic proof</h2>\n",
    "            <center>\n",
    "                <img src=\"https://wikidocs.net/images/page/178172/Fig_08_06_multi_head.gif\" width=\"400\" height=\"200\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\"/>\n",
    "                <p>this is implemented as several attention mechanisms whose outcomes are combined.\n",
    "The implementation simply splits the queries, keys, and values ​​it computes into parts for single-head attention. In this way, models with one or more attention heads are the same size. Multi-head attention does not increase model size.</p>\n",
    "            </center>   \n",
    "            <strong>Proof : </strong>\n",
    "<p>Let $\\mathbf{E} = [\\mathbf{e_1}, \\mathbf{e_2}, \\ldots, \\mathbf{e_n}]$ be a sequence of $n$ embeddings.</p>\n",
    "    <p>We create $h$ sets of learnable projection matrices $\\mathbf{W_Q^h}$, $\\mathbf{W_K^h}$, and $\\mathbf{W_V^h}$, where $h$ is the number of attention heads.</p>\n",
    "    <p>For each head $h$, we compute the Query ($\\mathbf{Q^h}$), Key ($\\mathbf{K^h}$), and Value ($\\mathbf{V^h}$) matrices as follows:</p>\n",
    "    <ul>\n",
    "        <li>$\\mathbf{Q_i^h} = \\mathbf{W_Q^h} \\mathbf{e_i}$</li>\n",
    "        <li>$\\mathbf{K_i^h} = \\mathbf{W_K^h} \\mathbf{e_i}$</li>\n",
    "        <li>$\\mathbf{V_i^h} = \\mathbf{W_V^h} \\mathbf{e_i}$</li>\n",
    "    </ul>\n",
    "<p>We then compute the dot-product similarity between the Query and Key matrices:</p>\n",
    "<p>$$\\text{Similarity}(\\mathbf{Q^h}, \\mathbf{K^h}) = \\frac{\\mathbf{Q^h} \\mathbf{K^h}^T}{\\sqrt{d_k}}$$</code></pre>\n",
    "    <p>where $d_k$ is the dimension of the Key matrix.</p>\n",
    "    <p>We apply a softmax function to the Similarity matrix:</p>\n",
    "    <p>$$\\mathbf{S^h} = \\text{softmax}(\\text{Similarity}(\\mathbf{Q^h}, \\mathbf{K^h})) = \\text{softmax}(\\frac{\\mathbf{Q^h} \\mathbf{K^h}^T}{\\sqrt{d_k}})$$</code></pre>\n",
    "    <p>We compute the weighted sum of the Value matrix using the Softmax weights:</p>\n",
    "    <p>$$\\text{Weighted Sum}(\\mathbf{V^h}, \\mathbf{S^h}) = \\mathbf{S^h} \\mathbf{V^h}$$</code></pre>\n",
    "    <p>We concatenate the output matrices from all heads together:</p>\n",
    "    <p>$$\\text{Output} = [\\text{Weighted Sum}(\\mathbf{V^1}, \\mathbf{S^1}), \\text{Weighted Sum}(\\mathbf{V^2}, \\mathbf{S^2}), \\ldots, \\text{Weighted Sum}(\\mathbf{V^h}, \\mathbf{S^h})]$$</code></pre>\n",
    "    <p>This results in a matrix of size $n \\times d_v h$, where $d_v$ is the number of features in each Value vector.</p>\n",
    "         </div>\n",
    "    </div>\n",
    "</div>\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.838958Z",
     "iopub.status.busy": "2023-05-11T13:48:09.838601Z",
     "iopub.status.idle": "2023-05-11T13:48:09.848974Z",
     "shell.execute_reply": "2023-05-11T13:48:09.847538Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.838911Z"
    }
   },
   "outputs": [],
   "source": [
    "# in here all what we dissccu stay the same only\n",
    "# we split the input emdebbing in to multi-dim of each Q , K . Y \n",
    "# because the Attention have the poeweful compute in Parallel manner \n",
    "\n",
    "def multiHeads_Attention(input_embedding ,wieghtsMatrix_QKY , heads ,out_Wieght , mask=None):\n",
    "    B , seq_len , embed_size = input_embedding.shape\n",
    "    # we have dim input of B . seq_len , \n",
    "    # embed_size ==> B, seq_len , embed_size/ heads \n",
    "    #=> Swape axis into [batch , heads , seq_len , embe_size / heads ]\n",
    "    Query , Key, Value = np.split(input_embedding@wieghtsMatrix_QKY,3, axis=-1)\n",
    "    Query , Key, Value = [a.reshape(B , seq_len ,heads , (embed_size // heads)).swapaxes(1,2) for a in (Query , Key, Value)]\n",
    "    if mask is not None:\n",
    "        atten = Softmax(Key@Query.swapaxes(-1,-2) / np.sqrt(embed_size // heads) + mask ) \n",
    "        return (atten@Value).swapaxes(1,2).reshape(B , seq_len , embed_size)@out_Wieght , atten\n",
    "    else:\n",
    "        atten = Softmax(Key@Query.swapaxes(-1,-2) / np.sqrt(embed_size // heads)) \n",
    "        return (atten@Value).swapaxes(1,2).reshape(B , seq_len , embed_size)@out_Wieght , atten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.850907Z",
     "iopub.status.busy": "2023-05-11T13:48:09.850533Z",
     "iopub.status.idle": "2023-05-11T13:48:09.863129Z",
     "shell.execute_reply": "2023-05-11T13:48:09.862222Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.850863Z"
    }
   },
   "outputs": [],
   "source": [
    "## intialize the input \n",
    "batch,lenght_sequnece , embedding_size = 1,100 , 64 \n",
    "number_heads = 2\n",
    "# now we will stand out with single Attetion heads \n",
    "input_embedding = torch.randn(batch,lenght_sequnece, embedding_size)\n",
    "atten = nn.MultiheadAttention(embedding_size,number_heads, bias = False,batch_first=True)\n",
    "## intialzie the Mask \n",
    "Mask = torch.triu(-float(\"inf\")*torch.ones(lenght_sequnece,lenght_sequnece),1)       \n",
    "out_weight_average , attention = atten(input_embedding,input_embedding,input_embedding,attn_mask=Mask)                  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.865115Z",
     "iopub.status.busy": "2023-05-11T13:48:09.864362Z",
     "iopub.status.idle": "2023-05-11T13:48:09.875732Z",
     "shell.execute_reply": "2023-05-11T13:48:09.874371Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.865080Z"
    }
   },
   "outputs": [],
   "source": [
    "## Our Multi-Heads Attention \n",
    "WieghtMatrix = atten.in_proj_weight\n",
    "out_Wieght = atten.out_proj.weight\n",
    "out_wieghts_Averege , atten = multiHeads_Attention(input_embedding.numpy(),\n",
    "                                     WieghtMatrix.detach().numpy().T,\n",
    "                                     number_heads,\n",
    "                                     out_Wieght.detach().numpy().T,  \n",
    "                                     mask=Mask.numpy())\n",
    "                                     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Transfomers</h2>\n",
    "            <p>Now that you understand the main model components and the general idea, let's take a look at the full model.Intuitively, the model is exactly what we discussed before. In the encoder, the tokens communicate with each other and update their representation. In the decoder, the target token first looks at the previously generated target token, then the source, and finally updates its representation. This happens in multiple layers, usually 6.\n",
    "\n",
    "Let's take a closer look at the other model components. :</p>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_09_01.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\"/>\n",
    "            </center>\n",
    "            <strong>Notation : </strong><p> in Next section we will cover other building blocks componemts in TRANSFIMERS</p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Token Embedding</h2>\n",
    "            <p>to train the Transfomer model we need to convert the input of word in to vector representation embedding which is BERT model </p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:09.878490Z",
     "iopub.status.busy": "2023-05-11T13:48:09.877269Z",
     "iopub.status.idle": "2023-05-11T13:48:09.885543Z",
     "shell.execute_reply": "2023-05-11T13:48:09.884395Z",
     "shell.execute_reply.started": "2023-05-11T13:48:09.878441Z"
    }
   },
   "outputs": [],
   "source": [
    "class Embedding(nn.Module):\n",
    "    def __init__(self , seq_len , embed_size):\n",
    "        super(Embedding,self).__init__()\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            vocab_size: size of vocabulary\n",
    "            embed_dim: dimension of embeddings\n",
    "        \"\"\"\n",
    "        self.embedding = nn.Embedding(seq_len , embed_size)\n",
    "    def forward(self,x):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            x: input vector\n",
    "        Returns:\n",
    "            out: embedding vector\n",
    "        \"\"\"\n",
    "        embed = self.embedding(x)\n",
    "        return embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Positional embedding</h2>\n",
    "            <p>Transformers do not involve recurrence or convolution, so they do not know the order of the input tokens. Therefore, we need to explicitly tell the model where the token is. There are two sets of embeddings for this. These are tokens (as we always have) and positions (new embeddings needed for this model). Then the input representation of the token is the sum of its two embeddings: the token and the position.\n",
    "                \n",
    "Unlike RNNs, which recurrently process tokens of a sequence one by one, self-attention ditches sequential operations in favor of parallel computation. Note, however, that self-attention by itself does not preserve the order of the sequence. What do we do if it really matters that the model knows in which order the input sequence arrived?</p>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_09_05.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"200\"/>\n",
    "                <p></p>\n",
    "</center>\n",
    "            <center>\n",
    "                <p>Positional embeddings can be trained, but the authors of the paper found that fixed embeddings did not affect quality. The fixed positional encodings used by Transformer are: \\begin{split}\\begin{aligned} p_{i, 2j} &= \\sin\\left(\\frac{i}{10000^{2j/d}}\\right),\\\\p_{i, 2j+1} &= \\cos\\left(\\frac{i}{10000^{2j/d}}\\right).\\end{aligned}\\end{split}</p>\n",
    "            </center>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:32:32.476080Z",
     "iopub.status.busy": "2023-05-11T16:32:32.475630Z",
     "iopub.status.idle": "2023-05-11T16:32:32.487313Z",
     "shell.execute_reply": "2023-05-11T16:32:32.485868Z",
     "shell.execute_reply.started": "2023-05-11T16:32:32.476041Z"
    }
   },
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    \"\"\"\n",
    "    compute sinusoid encoding.\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model, max_len):\n",
    "        super(PositionalEncoding,self).__init__()    \n",
    "        \"\"\"\n",
    "        constructor of sinusoid encoding class\n",
    "\n",
    "        :param d_model: dimension of model\n",
    "        :param max_len: max sequence length\n",
    "        :param device: hardware device setting\n",
    "        \"\"\"\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "\n",
    "        # same size with input matrix (for adding with input matrix)\n",
    "        self.encoding = torch.zeros(max_len, d_model)\n",
    "        self.encoding.requires_grad = False  # we don't need to compute gradient\n",
    "\n",
    "        pos = torch.arange(0, max_len)\n",
    "        pos = pos.float().unsqueeze(dim=1)\n",
    "        # 1D => 2D unsqueeze to represent word's position\n",
    "\n",
    "        _2i = torch.arange(0, d_model, step=2).float()\n",
    "        # 'i' means index of d_model (e.g. embedding size = 50, 'i' = [0,50])\n",
    "        # \"step=2\" means 'i' multiplied with two (same with 2 * i)\n",
    "\n",
    "        self.encoding[:, 0::2] = torch.sin(pos / (10000 ** (_2i / d_model)))\n",
    "        self.encoding[:, 1::2] = torch.cos(pos / (10000 ** (_2i / d_model)))\n",
    "        # compute positional encoding to consider positional information of words\n",
    "\n",
    "    def forward(self, x):\n",
    "        # self.encoding\n",
    "        # [max_len = 512, d_model = 512]\n",
    "\n",
    "        batch_size, seq_len = x.size()\n",
    "        # [batch_size = 128, seq_len = 30]\n",
    "\n",
    "        return self.encoding[:seq_len, :]\n",
    "        # [seq_len = 30, d_model = 512]\n",
    "        # it will add with tok_emb : [128, 30, 512]         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:19:49.764884Z",
     "iopub.status.busy": "2023-05-11T15:19:49.764480Z",
     "iopub.status.idle": "2023-05-11T15:19:49.772430Z",
     "shell.execute_reply": "2023-05-11T15:19:49.771098Z",
     "shell.execute_reply.started": "2023-05-11T15:19:49.764848Z"
    }
   },
   "outputs": [],
   "source": [
    "## intialize the input \n",
    "batch , max_len , encoding_dim = 1,100 , 100\n",
    "pos_embedding = torch.randn(batch,max_len, encoding_dim)\n",
    "pos_encoding = PositionalEncoding(encoding_dim, max_len)\n",
    "pos = pos_encoding(torch.zeros((batch, max_len)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:19:51.414767Z",
     "iopub.status.busy": "2023-05-11T15:19:51.414341Z",
     "iopub.status.idle": "2023-05-11T15:19:51.708124Z",
     "shell.execute_reply": "2023-05-11T15:19:51.707008Z",
     "shell.execute_reply.started": "2023-05-11T15:19:51.414728Z"
    }
   },
   "outputs": [],
   "source": [
    "fig , axe = plt.subplots(1 , 1 , figsize=(6,6))\n",
    "axe.set_title(\"Positional Encoding + Embedding input\")\n",
    "axe.set_xlabel(\"Key\")\n",
    "axe.set_ylabel(\"Query\")\n",
    "im =axe.imshow(pos, cmap=\"Blues\",interpolation=\"nearest\")\n",
    "divider = make_axes_locatable(axe)\n",
    "cax = divider.append_axes(\"right\", size=\"4%\", pad=0.2)\n",
    "cbar = plt.colorbar(im, cax=cax)\n",
    "fig.canvas.draw()\n",
    "fig.suptitle(\"Positional Embedding to Feed into Encoder and Decoder\")\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Residual connections </h2>\n",
    "            <p>We already saw residual connections when talking about convolutional language models. Residual connections are very simple (add a block's input to its output), but at the same time very useful. You can smooth out the gradient flow through the network and stack many layers.\n",
    "\n",
    "In Transformer, residual connections ((residual connections)) are used after each attention() and FFN block. In the picture above, the residuals are indicated by arrows coming around the block for the yellow \"Add & Norm\" layer. In the \"Add & Norm\" section, the \"Add\" section represents the residual connection</p>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_09_03.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"200\"/></center>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.249306Z",
     "iopub.status.busy": "2023-05-11T13:48:10.248960Z",
     "iopub.status.idle": "2023-05-11T13:48:10.255562Z",
     "shell.execute_reply": "2023-05-11T13:48:10.254616Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.249270Z"
    }
   },
   "outputs": [],
   "source": [
    "class ResidualCennection(nn.Module):\n",
    "    def __init__(self,x, residual):\n",
    "        super(ResidualCennection,self).__init__()\n",
    "        self.pass_trough = x\n",
    "        self.addtion = residual\n",
    "    def forward(self,*args, **kwargs):\n",
    "        x = self.pass_trough\n",
    "        return x + self.addtion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Positionwise Feed Forward</h2>\n",
    "            <p>In addition to attention, each layer has a feedforward network block. There is a ReLU nonlinearity between the two linear layers. After looking at other tokens via the attention mechanism, the model processes this new information using the FFN block (attention - \"gathers information by looking at other tokens\",FFN \"Take time to think and process this information\")</p>\n",
    "            <center><img src=\"https://wikidocs.net/images/page/178172/Fig_09_02.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"200\"/>\n",
    "                $$FFN(x) = \\max(0, xW_1+b_1)W_2+b_2.$$\n",
    "            </center>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.257666Z",
     "iopub.status.busy": "2023-05-11T13:48:10.256849Z",
     "iopub.status.idle": "2023-05-11T13:48:10.268257Z",
     "shell.execute_reply": "2023-05-11T13:48:10.267086Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.257628Z"
    }
   },
   "outputs": [],
   "source": [
    "class FeedForwrdNetwork(nn.Module):\n",
    "    def __init__(self,embed_size , hidden_size , dropout_pro=0.1):\n",
    "        super(FeedForwrdNetwork,self).__init__()\n",
    "        self.Linear_1= nn.Linear(embed_size,hidden_size)\n",
    "        self.Linear_2= nn.Linear(hidden_size,embed_size)\n",
    "        self.act = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(p=dropout_pro)\n",
    "    def forward(self,x):\n",
    "        x = self.Linear_1(x)\n",
    "        x = self.act(x)\n",
    "        x = self.Linear_2(x)\n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\">Layer Normalization</h2>\n",
    "            <p>The \"Norm\" part of the \"Add & Norm\" layer Layer Normalization represents . Normalize the vector representation of each example collectively and independently. This is done to control the \"flow/flow\" to the next layer. Layer normalization improves convergence stability and sometimes quality.\n",
    "In Transformer, we need to normalize the vector representation of each token. Also here LayerNorm is a learnable parameter that is used after normalization to rescale the layer's output (or the next layer's input).\n",
    "and There is. and is evaluated for each example, but and is the same. This is a layer parameter. </p>\n",
    "            <center><img src=\"https://github.com/hyunwoongko/transformer/raw/master/image/layer_norm.jpg\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"500\" height=\"300\"/></center>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:12:53.710883Z",
     "iopub.status.busy": "2023-05-11T15:12:53.710421Z",
     "iopub.status.idle": "2023-05-11T15:12:53.719632Z",
     "shell.execute_reply": "2023-05-11T15:12:53.718148Z",
     "shell.execute_reply.started": "2023-05-11T15:12:53.710841Z"
    }
   },
   "outputs": [],
   "source": [
    "class NormLayer(nn.Module):\n",
    "    def __init__(self,embed_size , eps=1e-12):\n",
    "        super(NormLayer, self).__init__()\n",
    "        self.embed_size = embed_size\n",
    "        self.gamma = nn.Parameter(torch.ones(embed_size))\n",
    "        self.beta = nn.Parameter(torch.zeros(embed_size))\n",
    "        self.eps = eps\n",
    "    def forward(self,x):\n",
    "        mean = x.mean(-1, keepdims= True)\n",
    "        var = x.var(-1 , unbiased=False , keepdims= True)\n",
    "        out = ( x - mean ) / torch.sqrt(var + self.eps)\n",
    "        out = self.gamma * out + self.beta\n",
    "        return out"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\"> Numpy Version Transfomers</h2>\n",
    "            <p>Now that you understand the main model components and the general idea, let's take a look at how to build the Full Transfomers model by combining all the componentes al once using Numpy</p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\"> Final Stage put all the pieces together to build Transfomers</h2>\n",
    "            <p>Now that you understand the main model components and the general idea, let's take a look at how to build the Full Transfomers model by combining all the componentes al once .\n",
    "\n",
    "Let's take a closer look at the other model components. :</p>\n",
    "            <center><img src=\"https://caisplusplus.usc.edu/images/curriculum/neural-network-flavors/transformers/transformer.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"300\"/>\n",
    "            </center>\n",
    "            <strong>Notation : </strong><p> in Next section we will cover other building FULL componemts in TRANSFIMERS</p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.282128Z",
     "iopub.status.busy": "2023-05-11T13:48:10.281759Z",
     "iopub.status.idle": "2023-05-11T13:48:10.291876Z",
     "shell.execute_reply": "2023-05-11T13:48:10.290732Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.282093Z"
    }
   },
   "outputs": [],
   "source": [
    "def NormLayar(Z):\n",
    "    mean = Z.mean(axis=-1 , keepdims=True)\n",
    "    var = Z.var(axis=-1 , keepdims = True)\n",
    "    return ((Z - mean) / np.sqrt(var)) + eps\n",
    "\n",
    "def ReLU(Z):\n",
    "    return np.maximum(Z,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.294104Z",
     "iopub.status.busy": "2023-05-11T13:48:10.293723Z",
     "iopub.status.idle": "2023-05-11T13:48:10.303643Z",
     "shell.execute_reply": "2023-05-11T13:48:10.302392Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.294067Z"
    }
   },
   "outputs": [],
   "source": [
    "## intialize the input \n",
    "batch,lenght_sequnece , embedding_size = 1,100 , 64 \n",
    "number_heads = 2\n",
    "# now we will stand out with single Attetion heads \n",
    "input_embedding = torch.randn(batch,lenght_sequnece, embedding_size)\n",
    "## intialzie the Mask \n",
    "Mask = torch.triu(-float(\"inf\")*torch.ones(lenght_sequnece,lenght_sequnece),1)       \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.305409Z",
     "iopub.status.busy": "2023-05-11T13:48:10.305087Z",
     "iopub.status.idle": "2023-05-11T13:48:10.331150Z",
     "shell.execute_reply": "2023-05-11T13:48:10.330172Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.305377Z"
    }
   },
   "outputs": [],
   "source": [
    "transEncoder = nn.TransformerEncoderLayer(embedding_size,number_heads,dim_feedforward=64,dropout=0.0,batch_first=True)\n",
    "output = transEncoder(input_embedding,Mask)\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.332537Z",
     "iopub.status.busy": "2023-05-11T13:48:10.332186Z",
     "iopub.status.idle": "2023-05-11T13:48:10.340347Z",
     "shell.execute_reply": "2023-05-11T13:48:10.339102Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.332501Z"
    }
   },
   "outputs": [],
   "source": [
    "## let's build first Transfomer Encode\n",
    "def TransfomerEncoder(embed_input,mask ,\n",
    "                      head, Wieghts_QKY , \n",
    "                      Wieghts_out ,FullyLinear1, \n",
    "                      FullLinear2 , eps):\n",
    "    input_embedding = embed_input.numpy()\n",
    "    multiHeads , _ = multiHeads_Attention(input_embedding,\n",
    "                                     Wieghts_QKY.detach().numpy().T,\n",
    "                                     head,\n",
    "                                     Wieghts_out.detach().numpy().T,  \n",
    "                                     mask=None)\n",
    "    Residual = NormLayar((input_embedding + multiHeads) + eps )\n",
    "    \n",
    "    output = NormLayar((Residual + ReLU(np.matmul(Residual,FullyLinear1))@FullLinear2) + eps)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.342635Z",
     "iopub.status.busy": "2023-05-11T13:48:10.342211Z",
     "iopub.status.idle": "2023-05-11T13:48:10.357198Z",
     "shell.execute_reply": "2023-05-11T13:48:10.355985Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.342590Z"
    }
   },
   "outputs": [],
   "source": [
    "WieghtMatrix = transEncoder.self_attn.in_proj_weight\n",
    "out_Wieght = transEncoder.self_attn.out_proj.weight\n",
    "Liearn1 = transEncoder.linear1.weight\n",
    "Linear2 = transEncoder.linear2.weight\n",
    "eps=1e-12\n",
    "output_ = TransfomerEncoder(input_embedding,Mask ,\n",
    "                      number_heads, WieghtMatrix , \n",
    "                      out_Wieght ,Liearn1.detach().numpy().T, \n",
    "                      Linear2.detach().numpy().T , eps)\n",
    "output_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.359554Z",
     "iopub.status.busy": "2023-05-11T13:48:10.359193Z",
     "iopub.status.idle": "2023-05-11T13:48:10.368837Z",
     "shell.execute_reply": "2023-05-11T13:48:10.367599Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.359519Z"
    }
   },
   "outputs": [],
   "source": [
    "# compare between TransfomerEncoerLayae Torch and Our \n",
    "np.linalg.norm(output_ - output.detach().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.371232Z",
     "iopub.status.busy": "2023-05-11T13:48:10.370766Z",
     "iopub.status.idle": "2023-05-11T13:48:10.381142Z",
     "shell.execute_reply": "2023-05-11T13:48:10.380142Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.371185Z"
    }
   },
   "outputs": [],
   "source": [
    "memory = torch.triu(-float(\"inf\")*torch.ones(lenght_sequnece,embedding_size),1).unsqueeze(1).transpose(1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.383333Z",
     "iopub.status.busy": "2023-05-11T13:48:10.382194Z",
     "iopub.status.idle": "2023-05-11T13:48:10.391049Z",
     "shell.execute_reply": "2023-05-11T13:48:10.389802Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.383280Z"
    }
   },
   "outputs": [],
   "source": [
    "memory.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.393868Z",
     "iopub.status.busy": "2023-05-11T13:48:10.393009Z",
     "iopub.status.idle": "2023-05-11T13:48:10.417442Z",
     "shell.execute_reply": "2023-05-11T13:48:10.416489Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.393819Z"
    }
   },
   "outputs": [],
   "source": [
    "transDecoder = nn.TransformerDecoderLayer(embedding_size ,number_heads , dim_feedforward=128 , batch_first=True)\n",
    "output_decoder = transDecoder(input_embedding,memory)\n",
    "output_decoder.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.419757Z",
     "iopub.status.busy": "2023-05-11T13:48:10.419079Z",
     "iopub.status.idle": "2023-05-11T13:48:10.425740Z",
     "shell.execute_reply": "2023-05-11T13:48:10.424524Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.419709Z"
    }
   },
   "outputs": [],
   "source": [
    "WieghtMatrix = transDecoder.self_attn.in_proj_weight\n",
    "out_Wieght = transDecoder.self_attn.out_proj.weight\n",
    "Liearn1 = transDecoder.linear1.weight\n",
    "Linear2 = transDecoder.linear2.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.428314Z",
     "iopub.status.busy": "2023-05-11T13:48:10.427553Z",
     "iopub.status.idle": "2023-05-11T13:48:10.444903Z",
     "shell.execute_reply": "2023-05-11T13:48:10.443237Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.428255Z"
    }
   },
   "outputs": [],
   "source": [
    "def TransfomerDecoder(embed_input,mask ,\n",
    "                      head, Wieghts_QKY , \n",
    "                      Wieghts_out ,FullyLinear1, \n",
    "                      FullLinear2 , eps,enc=None):\n",
    "    input_embedding = embed_input.numpy()\n",
    "    multiHeads , _ = multiHeads_Attention(input_embedding,\n",
    "                                     Wieghts_QKY.detach().numpy().T,\n",
    "                                     head,\n",
    "                                     Wieghts_out.detach().numpy().T,  \n",
    "                                     mask=None)\n",
    "    Residual = NormLayar((input_embedding + multiHeads) + eps )\n",
    "    \n",
    "    if enc is not None:\n",
    "        Query , key  = np.split(enc , 2 , axis=-1)\n",
    "        enc_ = np.concatenate((Query[:,:,:16] , key[:,:,:16]  ,Residual[:,:,:32] ), axis = -1)\n",
    "        MaskedMUltiHeads ,_= multiHeads_Attention(enc_,\n",
    "                                         Wieghts_QKY.detach().numpy().T,\n",
    "                                         head,\n",
    "                                         Wieghts_out.detach().numpy().T,  \n",
    "                                         mask=mask.numpy())\n",
    "        Residual = NormLayar((enc + MaskedMUltiHeads) + eps )\n",
    "    \n",
    "    output = NormLayar((Residual + ReLU(np.matmul(Residual,FullyLinear1))@FullLinear2) + eps)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.447900Z",
     "iopub.status.busy": "2023-05-11T13:48:10.447090Z",
     "iopub.status.idle": "2023-05-11T13:48:10.464258Z",
     "shell.execute_reply": "2023-05-11T13:48:10.462629Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.447847Z"
    }
   },
   "outputs": [],
   "source": [
    "output_decoder_ = TransfomerDecoder(input_embedding,Mask ,\n",
    "                      number_heads, WieghtMatrix , \n",
    "                      out_Wieght ,Liearn1.detach().numpy().T, \n",
    "                      Linear2.detach().numpy().T ,eps,enc=output_)\n",
    "output_decoder_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T13:48:10.466763Z",
     "iopub.status.busy": "2023-05-11T13:48:10.466045Z",
     "iopub.status.idle": "2023-05-11T13:48:10.476250Z",
     "shell.execute_reply": "2023-05-11T13:48:10.474701Z",
     "shell.execute_reply.started": "2023-05-11T13:48:10.466717Z"
    }
   },
   "outputs": [],
   "source": [
    "np.linalg.norm(output_decoder_ - output_decoder.detach().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:13:04.325576Z",
     "iopub.status.busy": "2023-05-11T15:13:04.324332Z",
     "iopub.status.idle": "2023-05-11T15:13:04.336153Z",
     "shell.execute_reply": "2023-05-11T15:13:04.334974Z",
     "shell.execute_reply.started": "2023-05-11T15:13:04.325527Z"
    }
   },
   "outputs": [],
   "source": [
    "class Transformer:\n",
    "    def __init__(self, embedding_size , heads):\n",
    "        super(Transformer,self).__init__()\n",
    "        \n",
    "        \"\"\"\n",
    "        Blocks models\n",
    "        -------------\n",
    "        encoder : is used without the Mask at this stage \n",
    "        Decoder : include the Mask and with trick to split the Qurey and Key \n",
    "        \n",
    "        Parametes :\n",
    "        Wieghst_QKY_Encoder : is Leanrble wieght matrix feed into Encoder\n",
    "        Wieghst_QKY_Decoder : is Leanrble wieght matrix feed into ncoder\n",
    "        embed_size : is mebeding_size dimession of input \n",
    "        Seq_len : max lenght of vacolublaries \n",
    "        Linear_1 : Linear Denes layar of Deooder\n",
    "        Linear_2 : Linear Denes layar of Deooder after include the ooutput from Encoder\n",
    "        Linear_encoder : Linear Denes layar of Enooder\n",
    "        \"\"\"\n",
    "    \n",
    "        self.embed_input = embedding_size\n",
    "        self.heads = heads \n",
    "        self.eps = 1e-12\n",
    "        self.W_QKY_Encoder = transEncoder.self_attn.in_proj_weight\n",
    "        self.W_Out_Encoder = transEncoder.self_attn.out_proj.weight\n",
    "        self.W_QKY_Decoder = transDecoder.self_attn.in_proj_weight\n",
    "        self.W_Out_Decoder = transDecoder.self_attn.out_proj.weight\n",
    "        self.Linear1_Encoder = transEncoder.linear1.weight\n",
    "        self.Linear2_Encoder = transEncoder.linear2.weight\n",
    "        self.Linear1_Decoder = transDecoder.linear1.weight\n",
    "        self.Linear2_Decoder = transDecoder.linear2.weight\n",
    "    \n",
    "    def forward(self, enc_ , dec_, mask):\n",
    "        Encoder_= TransfomerEncoder(enc_ ,None ,\n",
    "                  self.heads, self.W_QKY_Encoder , \n",
    "                  self.W_Out_Encoder ,self.Linear1_Encoder.detach().numpy().T, \n",
    "                  self.Linear2_Encoder.detach().numpy().T , self.eps)\n",
    "\n",
    "        Decoder_ = TransfomerDecoder(dec_, mask ,\n",
    "                  self.heads, self.W_QKY_Decoder , \n",
    "                  self.W_Out_Decoder , self.Linear1_Decoder.detach().numpy().T, \n",
    "                  self.Linear2_Decoder.detach().numpy().T ,self.eps,enc=Encoder_)\n",
    "\n",
    "        return Decoder_\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:13:07.129839Z",
     "iopub.status.busy": "2023-05-11T15:13:07.129445Z",
     "iopub.status.idle": "2023-05-11T15:13:07.155512Z",
     "shell.execute_reply": "2023-05-11T15:13:07.154138Z",
     "shell.execute_reply.started": "2023-05-11T15:13:07.129804Z"
    }
   },
   "outputs": [],
   "source": [
    "## intialize the input \n",
    "batch,lenght_sequnece , embedding_size = 1,100 , 64 \n",
    "heads = 2\n",
    "# now we will stand out with single Attetion heads \n",
    "input_embedding = torch.randn(batch,lenght_sequnece, embedding_size)\n",
    "## intialzie the Mask \n",
    "Mask = torch.triu(-float(\"inf\")*torch.ones(lenght_sequnece,lenght_sequnece),1)       \n",
    "\n",
    "Trans = Transformer(embedding_size,heads)\n",
    "output=Trans.forward(input_embedding,input_embedding,Mask)\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\"> Torch Version Transfomers</h2>\n",
    "            <p>Now that you built Numpy version from Scratch, in DL actually we will use Pytorch and we wil do training of the model Translation , now let us build Transfomers , only we do is Combine the componentes of Transfomers we built Already  </p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background-color: #f5f5f5; padding: 10px; border-radius: 5px;\">\n",
    "    <div class=\"card mt-3\">\n",
    "        <div class=\"card-body\">\n",
    "            <h2 class=\"card-title text-center mb-4\"> Final Stage put all the pieces together to build Transfomers</h2>\n",
    "            <p>Now that you understand the main model components and the general idea, let's take a look at how to build the Full Transfomers model by combining all the componentes al once .\n",
    "\n",
    "Let's take a closer look at the other model components. :</p>\n",
    "            <center><img src=\"https://caisplusplus.usc.edu/images/curriculum/neural-network-flavors/transformers/transformer.png\" title=\"Attention(Q,K,V)=softmax(QKT/sqrt(d_k))V\" style=\"background-color:gray; padding: 10px; border-radius: 5px;\" width=\"400\" height=\"300\"/>\n",
    "            </center>\n",
    "            <strong>Notation : </strong><p> in Next section we will cover other building FULL componemts in TRANSFIMERS</p>\n",
    "        </div>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:13:10.737094Z",
     "iopub.status.busy": "2023-05-11T15:13:10.736654Z",
     "iopub.status.idle": "2023-05-11T15:13:10.746288Z",
     "shell.execute_reply": "2023-05-11T15:13:10.744589Z",
     "shell.execute_reply.started": "2023-05-11T15:13:10.737055Z"
    }
   },
   "outputs": [],
   "source": [
    "class ScaleProduct(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ScaleProduct, self).__init__()\n",
    "        self.Softmax = nn.Softmax(dim=-1)\n",
    "    def forward(self,query , key , value , mask = None , eps = 1e-12):\n",
    "        batch , heads , seq_len , embed_size_ = key.size()\n",
    "        # product query , key \n",
    "        key_t = key.transpose(2,3)\n",
    "        dot_product = (query@key_t) / math.sqrt(embed_size_)\n",
    "        \n",
    "        if mask is not None: \n",
    "            Score = dot_product.masked_fill(mask == 0,-10000)\n",
    "        \n",
    "        compute_similarty = self.Softmax(Score)\n",
    "        \n",
    "        value = compute_similarty @ value\n",
    "        \n",
    "        return value , Score \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T15:13:12.183553Z",
     "iopub.status.busy": "2023-05-11T15:13:12.182725Z",
     "iopub.status.idle": "2023-05-11T15:13:12.195446Z",
     "shell.execute_reply": "2023-05-11T15:13:12.194370Z",
     "shell.execute_reply.started": "2023-05-11T15:13:12.183507Z"
    }
   },
   "outputs": [],
   "source": [
    "## Let us Build first multi-Heads-Attention\n",
    "\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self,embed_size , heads):\n",
    "        super(MultiHeadAttention,self).__init__()\n",
    "        self.heads = heads \n",
    "        self.Query = nn.Linear(embed_size ,embed_size)\n",
    "        self.Key = nn.Linear(embed_size ,embed_size)\n",
    "        self.Value = nn.Linear(embed_size ,embed_size)\n",
    "        self.concat = nn.Linear(embed_size ,embed_size)\n",
    "        self.Scale_product = ScaleProduct()\n",
    "    \n",
    "    def forward(self , query , key , value , mask=None):\n",
    "        query , key , value = self.Query(query),self.Key(key) , self.Value(value)\n",
    "        query_dim , key_dim , value_dim = self.split_dim(query),self.split_dim(key),self.split_dim(value)        \n",
    "        Socore , _ = self.Scale_product(query_dim , key_dim , value_dim , mask = mask)\n",
    "        concating_dim = self.concating(Socore)\n",
    "        out = self.concat(concating_dim)\n",
    "        return out\n",
    "    def split_dim(self , tensor):\n",
    "        \"\"\"\n",
    "        input_dim ==> [batch , seq_len , embed_size]\n",
    "        split into sub-dims ==> [bacth , seq_len , heads , embed_size / numberHeads ]\n",
    "        where : \n",
    "            INTERSTRED DIMES ARE SEQ_LEN AND EMBED_SIZE \n",
    "            WE WILL TARSNSPOSE THEM BETWEEN DIMS \n",
    "        \"\"\"\n",
    "        batch , seq_len , embed_size = tensor.size()\n",
    "        \n",
    "        tensor = tensor.view(batch,seq_len , heads , embed_size // self.heads).transpose(1,2)\n",
    "        return tensor\n",
    "    \n",
    "    def concating(self, tensor):\n",
    "        bacth , heads , seq_len , embed_size = tensor.size()\n",
    "        tensor = tensor.transpose(2,1).contiguous()\n",
    "        tensor = tensor.reshape(batch , seq_len , heads*embed_size)\n",
    "        return tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:33:14.945776Z",
     "iopub.status.busy": "2023-05-11T16:33:14.945336Z",
     "iopub.status.idle": "2023-05-11T16:33:14.952354Z",
     "shell.execute_reply": "2023-05-11T16:33:14.951261Z",
     "shell.execute_reply.started": "2023-05-11T16:33:14.945735Z"
    }
   },
   "outputs": [],
   "source": [
    "class Embedding(nn.Module):\n",
    "    def __init__(self,seq_len,embed_size):\n",
    "        super(Embedding,self).__init__()\n",
    "        self.embedding_input= nn.Embedding(embed_size,seq_len)\n",
    "    def forward(self , x ):\n",
    "        x = self.embedding_input(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:35:41.613479Z",
     "iopub.status.busy": "2023-05-11T16:35:41.613044Z",
     "iopub.status.idle": "2023-05-11T16:35:41.620984Z",
     "shell.execute_reply": "2023-05-11T16:35:41.619806Z",
     "shell.execute_reply.started": "2023-05-11T16:35:41.613441Z"
    }
   },
   "outputs": [],
   "source": [
    "class TokenPositional(nn.Module):\n",
    "    def __init__(self,embed_size,seq_len,drop_pro):\n",
    "        super(TokenPositional,self).__init__()\n",
    "        self.Positional = PositionalEncoding(seq_len,embed_size)\n",
    "        self.Embeddding = Embedding(seq_len,embed_size)\n",
    "        self.Dropout = nn.Dropout(drop_pro)\n",
    "    def forward(self,x):\n",
    "        emd =  self.Embeddding(x)\n",
    "        posi = self.Positional(x)\n",
    "        return self.Dropout(emd+ posi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:38:56.877468Z",
     "iopub.status.busy": "2023-05-11T16:38:56.877000Z",
     "iopub.status.idle": "2023-05-11T16:38:56.887088Z",
     "shell.execute_reply": "2023-05-11T16:38:56.885712Z",
     "shell.execute_reply.started": "2023-05-11T16:38:56.877423Z"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self, embed_size , heads, hidden_dim , drop_pro):\n",
    "        super(EncoderLayer,self).__init__()\n",
    "        self.MultiHeadAttention = MultiHeadAttention(embed_size , heads)\n",
    "        self.NormLayer1= NormLayer(embed_size)\n",
    "        self.NormLayer2= NormLayer(embed_size)\n",
    "        self.FeedForward = FeedForwrdNetwork(embed_size,hidden_dim)\n",
    "        self.Dropout_1 = nn.Dropout(drop_pro)\n",
    "        self.Dropout_2 = nn.Dropout(drop_pro)\n",
    "\n",
    "    def forward(self, x,mask):\n",
    "        x_ = x\n",
    "        x = self.MultiHeadAttention(query=x , key=x , value=x , mask=mask)\n",
    "        x = self.Dropout_1(x)\n",
    "        \n",
    "        x = self.NormLayer1(x_ + x)\n",
    "        print(x.shape)\n",
    "       \n",
    "        x_ = x \n",
    "\n",
    "        x = self.FeedForward(x)\n",
    "        fnn = self.Dropout_2(x)\n",
    "        x = self.NormLayer2(x + x_)\n",
    "        return x      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:41:01.673447Z",
     "iopub.status.busy": "2023-05-11T16:41:01.672991Z",
     "iopub.status.idle": "2023-05-11T16:41:01.682064Z",
     "shell.execute_reply": "2023-05-11T16:41:01.680612Z",
     "shell.execute_reply.started": "2023-05-11T16:41:01.673405Z"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self,embed_size , seq_len , heads , hidden_dim, drop_pro, StackLayers):\n",
    "        super(Encoder,self).__init__()\n",
    "        self.InputEmbedding = TokenPositional(embed_size,\n",
    "                                              seq_len,\n",
    "                                              drop_pro)\n",
    "        self.EncoderLayer = nn.ModuleList([EncoderLayer(embed_size , heads, hidden_dim , drop_pro) \n",
    "                                          for _ in range(StackLayers)])\n",
    "        \n",
    "    def forward(self ,x , src_mask):\n",
    "        x = self.InputEmbedding(x)\n",
    "        for layer in self.EncoderLayer:\n",
    "            x - layer(x , src_mask)\n",
    "            return x     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T14:20:25.080469Z",
     "iopub.status.busy": "2023-05-11T14:20:25.079285Z",
     "iopub.status.idle": "2023-05-11T14:20:25.092287Z",
     "shell.execute_reply": "2023-05-11T14:20:25.091004Z",
     "shell.execute_reply.started": "2023-05-11T14:20:25.080408Z"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self , embed_size , heads , hidden_size ,drop_proba=0.1):\n",
    "        super(DecoderLayer,self).__init__()\n",
    "        self.mulitHeads = MultiHeadAttention(embed_size , heads)\n",
    "        self.Norm1 = NormLayer(embed_size)\n",
    "        self.dropout1 = nn.Dropout(p=drop_proba)\n",
    "        \n",
    "        self.mulitHead_enc = MultiHeadAttention(embed_size , heads)\n",
    "        self.Norm2= NormLayer(embed_size)\n",
    "        self.dropout2 = nn.Dropout(p=drop_proba)\n",
    "        \n",
    "        self.fnn = FeedForwrdNetwork(embed_size , hidden_size)\n",
    "        self.Norm3 = NormLayer(embde_size)\n",
    "        self.dropout3 = nn.Dropout(p=drop_proba)\n",
    "    def forward(self ,dec , enc , trg_mask , src_mask):\n",
    "        # compute attention on decoder \n",
    "        x_ = x \n",
    "        x = self.mulitHeadxs(query= x , key = x , value = x , mask = trg_mask)\n",
    "        x = self.dropout1(x)\n",
    "        x = self.Norm1(x + x_)\n",
    "        \n",
    "        if enc is not None:\n",
    "            x_ = x\n",
    "            x = self.mulitHead_enc(query = x_ , key = enc , value =enc , mask= mask_src)\n",
    "            x = self.dropout2(x)\n",
    "            x = self.Norm2( x + x_)\n",
    "        \n",
    "        x_ = x \n",
    "        \n",
    "        x = self.fnn(x)\n",
    "        x = self.droout3(x)\n",
    "        x = self.Norm3(x + x_ )\n",
    "        return x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T14:26:44.514684Z",
     "iopub.status.busy": "2023-05-11T14:26:44.514203Z",
     "iopub.status.idle": "2023-05-11T14:26:44.524037Z",
     "shell.execute_reply": "2023-05-11T14:26:44.522911Z",
     "shell.execute_reply.started": "2023-05-11T14:26:44.514614Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self , emdbe_size ,seq_len , heeds , hidden_Size ,drop_proba =0.1 ):\n",
    "        super(Decoder , self).__init__()\n",
    "        self.InputEmbedding = TokenPositional(embed_size,\n",
    "                                              seq_len,\n",
    "                                              drop_pro, \n",
    "                                              device)\n",
    "        self.Layer = nn.ModuleDict([DecoderLayer(embed_size, \n",
    "                                                 heads , \n",
    "                                                 hidden_Size) \n",
    "                                    for _ in range(n_layers)])\n",
    "        self.Linear = nn.Linear(embed_size , embed_size)\n",
    "    def forward(self, dec , enc , trg_mask , src_mask):\n",
    "        x = self.InputEmbedding(dec)\n",
    "        for layer in self.Layer:\n",
    "            trg = layer(x , dec , trg_mask , src_mask )\n",
    "        \n",
    "        out - self.Linear(out)\n",
    "        return out       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-11T16:43:11.635144Z",
     "iopub.status.busy": "2023-05-11T16:43:11.634625Z",
     "iopub.status.idle": "2023-05-11T16:43:11.650605Z",
     "shell.execute_reply": "2023-05-11T16:43:11.649220Z",
     "shell.execute_reply.started": "2023-05-11T16:43:11.635102Z"
    }
   },
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "\n",
    "    def __init__(self, src_pad_idx, trg_pad_idx, trg_sos_idx, enc_voc_size, dec_voc_size, d_model, n_head, max_len,\n",
    "                 ffn_hidden, n_layers, drop_prob, device):\n",
    "        super().__init__()\n",
    "        self.src_pad_idx = src_pad_idx\n",
    "        self.trg_pad_idx = trg_pad_idx\n",
    "        self.trg_sos_idx = trg_sos_idx\n",
    "        self.device = device\n",
    "        self.encoder = Encoder(d_model=d_model,\n",
    "                               n_head=n_head,\n",
    "                               max_len=max_len,\n",
    "                               ffn_hidden=ffn_hidden,\n",
    "                               enc_voc_size=enc_voc_size,\n",
    "                               drop_prob=drop_prob,\n",
    "                               n_layers=n_layers,\n",
    "                               device=device)\n",
    "\n",
    "        self.decoder = Decoder(d_model=d_model,\n",
    "                               n_head=n_head,\n",
    "                               max_len=max_len,\n",
    "                               ffn_hidden=ffn_hidden,\n",
    "                               dec_voc_size=dec_voc_size,\n",
    "                               drop_prob=drop_prob,\n",
    "                               n_layers=n_layers,\n",
    "                               device=device)\n",
    "\n",
    "    def forward(self, src, trg):\n",
    "        src_mask = self.make_pad_mask(src, src, self.src_pad_idx, self.src_pad_idx)\n",
    "\n",
    "        src_trg_mask = self.make_pad_mask(trg, src, self.trg_pad_idx, self.src_pad_idx)\n",
    "\n",
    "        trg_mask = self.make_pad_mask(trg, trg, self.trg_pad_idx, self.trg_pad_idx) * \\\n",
    "                   self.make_no_peak_mask(trg, trg)\n",
    "\n",
    "        enc_src = self.encoder(src, src_mask)\n",
    "        output = self.decoder(trg, enc_src, trg_mask, src_trg_mask)\n",
    "        return output\n",
    "\n",
    "    def make_pad_mask(self, q, k, q_pad_idx, k_pad_idx):\n",
    "        len_q, len_k = q.size(1), k.size(1)\n",
    "\n",
    "        # batch_size x 1 x 1 x len_k\n",
    "        k = k.ne(k_pad_idx).unsqueeze(1).unsqueeze(2)\n",
    "        # batch_size x 1 x len_q x len_k\n",
    "        k = k.repeat(1, 1, len_q, 1)\n",
    "\n",
    "        # batch_size x 1 x len_q x 1\n",
    "        q = q.ne(q_pad_idx).unsqueeze(1).unsqueeze(3)\n",
    "        # batch_size x 1 x len_q x len_k\n",
    "        q = q.repeat(1, 1, 1, len_k)\n",
    "\n",
    "        mask = k & q\n",
    "        return mask\n",
    "\n",
    "    def make_no_peak_mask(self, q, k):\n",
    "        len_q, len_k = q.size(1), k.size(1)\n",
    "\n",
    "        # len_q x len_k\n",
    "        mask = torch.tril(torch.ones(len_q, len_k)).type(torch.BoolTensor).to(self.device)\n",
    "\n",
    "        return mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center>\n",
    "    <div class=\"alert alert-danger\" role=\"alert\"> <strong>this notebook still under progress Transfomer in Depth</strong>\n",
    "    </div>\n",
    "<img src=\"https://img.shields.io/badge/Upvote-If%20you%20like%20my%20work-07b3c8?style=for-the-badge&logo=kaggle\">\n",
    "</center>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 insatall "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}