1.0 Data Ingestion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-01T04:14:36.759514Z",
     "start_time": "2022-01-01T04:14:13.425865Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Requirements.txt already exists, use --force to overwrite it\n"
     ]
    }
   ],
   "source": [
    "!pipreqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-31T11:52:19.500070Z",
     "start_time": "2021-07-31T11:52:19.451963Z"
    },
    "executionInfo": {
     "elapsed": 60,
     "status": "ok",
     "timestamp": 1638198672688,
     "user": {
      "displayName": "Sarthak Rastogi",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgjV2l0bM183bdefu3PYpY3V0EEWbr7EAUJZfSsQyA=s64",
      "userId": "13011506269675720653"
     },
     "user_tz": -330
    },
    "id": "FxGyDiCe3J2t"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-r-Bla476V0X"
   },
   "source": [
    "# Importing raw data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xRXG0qZL4kjI"
   },
   "source": [
    "Data: https://nlp.stanford.edu/data/glove.6B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-31T11:51:52.431554Z",
     "start_time": "2021-07-31T11:51:50.034Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 195963,
     "status": "ok",
     "timestamp": 1638198868597,
     "user": {
      "displayName": "Sarthak Rastogi",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgjV2l0bM183bdefu3PYpY3V0EEWbr7EAUJZfSsQyA=s64",
      "userId": "13011506269675720653"
     },
     "user_tz": -330
    },
    "id": "iJQsATTRA824",
    "outputId": "889b2ef7-6490-44db-ac8b-4502d3ad8589"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "project_path = \"/content/drive/MyDrive/Colab Notebooks/Projects/PIE/\"\n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-31T11:51:52.432550Z",
     "start_time": "2021-07-31T11:51:50.038Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 208,
     "referenced_widgets": [
      "57474e7923f04eefb8d7c716b1f633b1",
      "4af97c7e507343a893b2c873949ef505",
      "2e9c296432264b93aea89953907ff032",
      "4b0af7f31787402aa00fc608346d5b16",
      "bcb42848cc474c36b9adc745370ce643",
      "d2cbad6554874ded862d9939be6a39d9",
      "478eee00a6b74e40b8a89d7e6b869bd6",
      "7fa40c16199a4677892ca606ac69e24f",
      "d181c0da7fd54c4cbcb9dd158c78aa0f",
      "b26fb8bb4f654b2abc7819ca65d7691f",
      "c4f2fdc4a01f46a48182ca24194076b0"
     ]
    },
    "executionInfo": {
     "elapsed": 14038,
     "status": "ok",
     "timestamp": 1638198882618,
     "user": {
      "displayName": "Sarthak Rastogi",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgjV2l0bM183bdefu3PYpY3V0EEWbr7EAUJZfSsQyA=s64",
      "userId": "13011506269675720653"
     },
     "user_tz": -330
    },
    "id": "l431vC5Z3Qpy",
    "outputId": "f938dd2f-73f1-4e80-bf7c-f26f86b4ea5c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the 0.418 0.24968 -0.41242 0.1217 0.34527 -0.044457 -0.49688 -0.17862 -0.00066023 -0.6566 0.27843 -0.14767 -0.55677 0.14658 -0.0095095 0.011658 0.10204 -0.12792 -0.8443 -0.12181 -0.016801 -0.33279 -0.1552 -0.23131 -0.19181 -1.8823 -0.76746 0.099051 -0.42125 -0.19526 4.0071 -0.18594 -0.52287 -0.31681 0.00059213 0.0074449 0.17778 -0.15897 0.012041 -0.054223 -0.29871 -0.15749 -0.34758 -0.045637 -0.44251 0.18785 0.0027849 -0.18411 -0.11514 -0.78581\n",
      "\n",
      ", 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 -0.42852 -0.55641 -0.364 -0.23938 0.13001 -0.063734 -0.39575 -0.48162 0.23291 0.090201 -0.13324 0.078639 -0.41634 -0.15428 0.10068 0.48891 0.31226 -0.1252 -0.037512 -1.5179 0.12612 -0.02442 -0.042961 -0.28351 3.5416 -0.11956 -0.014533 -0.1499 0.21864 -0.33412 -0.13872 0.31806 0.70358 0.44858 -0.080262 0.63003 0.32111 -0.46765 0.22786 0.36034 -0.37818 -0.56657 0.044691 0.30392\n",
      "\n",
      ". 0.15164 0.30177 -0.16763 0.17684 0.31719 0.33973 -0.43478 -0.31086 -0.44999 -0.29486 0.16608 0.11963 -0.41328 -0.42353 0.59868 0.28825 -0.11547 -0.041848 -0.67989 -0.25063 0.18472 0.086876 0.46582 0.015035 0.043474 -1.4671 -0.30384 -0.023441 0.30589 -0.21785 3.746 0.0042284 -0.18436 -0.46209 0.098329 -0.11907 0.23919 0.1161 0.41705 0.056763 -6.3681e-05 0.068987 0.087939 -0.10285 -0.13931 0.22314 -0.080803 -0.35652 0.016413 0.10216\n",
      "\n",
      "of 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 0.18157 -0.52393 0.10381 -0.17566 0.078852 -0.36216 -0.11829 -0.83336 0.11917 -0.16605 0.061555 -0.012719 -0.56623 0.013616 0.22851 -0.14396 -0.067549 -0.38157 -0.23698 -1.7037 -0.86692 -0.26704 -0.2589 0.1767 3.8676 -0.1613 -0.13273 -0.68881 0.18444 0.0052464 -0.33874 -0.078956 0.24185 0.36576 -0.34727 0.28483 0.075693 -0.062178 -0.38988 0.22902 -0.21617 -0.22562 -0.093918 -0.80375\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57474e7923f04eefb8d7c716b1f633b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Run only once\n",
    "\n",
    "glove_file_path = \"/content/drive/MyDrive/Colab Notebooks/Projects/PIE/Embeddings/glove.6B.50d.txt\"\n",
    "\n",
    "with open(glove_file_path, 'r', encoding=\"utf-8\") as file:\n",
    "  print(file.readline())\n",
    "  print(file.readline())\n",
    "  print(file.readline())\n",
    "  print(file.readline())\n",
    "\n",
    "english_embeddings_dict = {}\n",
    "\n",
    "with open(glove_file_path, 'r', encoding=\"utf-8\") as file:\n",
    "    for i in tqdm(file):\n",
    "        values = i.split()\n",
    "        word = values[0]\n",
    "        vector = np.asarray(values[1:], 'float32')\n",
    "        english_embeddings_dict[word] = vector\n",
    "\n",
    "with open(project_path + \"Embeddings/english_embeddings.pickle\", \"wb\") as f:\n",
    "  pickle.dump(english_embeddings_dict, f, protocol=pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-07-31T11:51:52.433551Z",
     "start_time": "2021-07-31T11:51:50.042Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 182
    },
    "executionInfo": {
     "elapsed": 39,
     "status": "error",
     "timestamp": 1638198882633,
     "user": {
      "displayName": "Sarthak Rastogi",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgjV2l0bM183bdefu3PYpY3V0EEWbr7EAUJZfSsQyA=s64",
      "userId": "13011506269675720653"
     },
     "user_tz": -330
    },
    "id": "nEa92InHk-Ay",
    "outputId": "823db9b2-b15a-4d2c-ed9b-376b99d2930d"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-f490d44fbcbd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"Embeddings/english_embeddings.pickle\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0menglish_embeddings_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/MyDrive/Colab Notebooks/Projects/PIE/Embeddings/english_embeddings.pickle'"
     ]
    }
   ],
   "source": [
    "with open(project_path + \"Embeddings/english_embeddings.pickle\", 'rb') as f:\n",
    "    english_embeddings_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qq62W8zr3zje"
   },
   "source": [
    "The composition of the dictionary is as follows: corresponding to each word, there is an array of length matching the dimensions of the GloVe embeddings chosen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27,
     "status": "aborted",
     "timestamp": 1638198882629,
     "user": {
      "displayName": "Sarthak Rastogi",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgjV2l0bM183bdefu3PYpY3V0EEWbr7EAUJZfSsQyA=s64",
      "userId": "13011506269675720653"
     },
     "user_tz": -330
    },
    "id": "hupuq2-mtLTV"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "1.0 Data Ingestion.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "2e9c296432264b93aea89953907ff032": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_478eee00a6b74e40b8a89d7e6b869bd6",
      "placeholder": "​",
      "style": "IPY_MODEL_d2cbad6554874ded862d9939be6a39d9",
      "value": ""
     }
    },
    "478eee00a6b74e40b8a89d7e6b869bd6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4af97c7e507343a893b2c873949ef505": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4b0af7f31787402aa00fc608346d5b16": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d181c0da7fd54c4cbcb9dd158c78aa0f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7fa40c16199a4677892ca606ac69e24f",
      "value": 1
     }
    },
    "57474e7923f04eefb8d7c716b1f633b1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2e9c296432264b93aea89953907ff032",
       "IPY_MODEL_4b0af7f31787402aa00fc608346d5b16",
       "IPY_MODEL_bcb42848cc474c36b9adc745370ce643"
      ],
      "layout": "IPY_MODEL_4af97c7e507343a893b2c873949ef505"
     }
    },
    "7fa40c16199a4677892ca606ac69e24f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b26fb8bb4f654b2abc7819ca65d7691f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bcb42848cc474c36b9adc745370ce643": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c4f2fdc4a01f46a48182ca24194076b0",
      "placeholder": "​",
      "style": "IPY_MODEL_b26fb8bb4f654b2abc7819ca65d7691f",
      "value": " 400000/? [00:09&lt;00:00, 56197.39it/s]"
     }
    },
    "c4f2fdc4a01f46a48182ca24194076b0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d181c0da7fd54c4cbcb9dd158c78aa0f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "20px"
     }
    },
    "d2cbad6554874ded862d9939be6a39d9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}