Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://job.jobnet.dk/CV\n"
     ]
    },
    {
     "ename": "WebDriverException",
     "evalue": "Message: Failed to decode response from marionette\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchWindowException\u001b[0m                     Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-34-25589e085f9f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     43\u001b[0m     SearchField =  WebDriverWait(driver, 30).until(\n\u001b[1;32m---> 44\u001b[1;33m         \u001b[0mEC\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvisibility_of_element_located\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"SearchFieldAngular\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     45\u001b[0m     )\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\wait.py\u001b[0m in \u001b[0;36muntil\u001b[1;34m(self, method, message)\u001b[0m\n\u001b[0;32m     70\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 71\u001b[1;33m                 \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_driver\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     72\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\expected_conditions.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, driver)\u001b[0m\n\u001b[0;32m    126\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 127\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0m_element_if_visible\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_find_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlocator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    128\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStaleElementReferenceException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\expected_conditions.py\u001b[0m in \u001b[0;36m_find_element\u001b[1;34m(driver, by)\u001b[0m\n\u001b[0;32m    400\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mWebDriverException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 401\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    402\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\expected_conditions.py\u001b[0m in \u001b[0;36m_find_element\u001b[1;34m(driver, by)\u001b[0m\n\u001b[0;32m    396\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 397\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    398\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mNoSuchElementException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    954\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 955\u001b[1;33m             'value': value})['value']\n\u001b[0m\u001b[0;32m    956\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    311\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 312\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    313\u001b[0m             response['value'] = self._unwrap_value(\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchWindowException\u001b[0m: Message: Browsing context has been discarded\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-34-25589e085f9f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     45\u001b[0m     )\n\u001b[0;32m     46\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m     \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_screenshot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'screen.png'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m     \u001b[0mSearchField\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclear\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m     \u001b[0mSearchField\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_keys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Data\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36msave_screenshot\u001b[1;34m(self, filename)\u001b[0m\n\u001b[0;32m   1029\u001b[0m             \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_screenshot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'/Screenshots/foo.png'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1030\u001b[0m         \"\"\"\n\u001b[1;32m-> 1031\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_screenshot_as_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1032\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1033\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_screenshot_as_png\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mget_screenshot_as_file\u001b[1;34m(self, filename)\u001b[0m\n\u001b[0;32m   1006\u001b[0m             warnings.warn(\"name used for saved screenshot does not match file \"\n\u001b[0;32m   1007\u001b[0m                           \"type. It should end with a `.png` extension\", UserWarning)\n\u001b[1;32m-> 1008\u001b[1;33m         \u001b[0mpng\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_screenshot_as_png\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1009\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1010\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mget_screenshot_as_png\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1038\u001b[0m             \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_screenshot_as_png\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1039\u001b[0m         \"\"\"\n\u001b[1;32m-> 1040\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mbase64\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mb64decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_screenshot_as_base64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'ascii'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1042\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_screenshot_as_base64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mget_screenshot_as_base64\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1048\u001b[0m             \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_screenshot_as_base64\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1049\u001b[0m         \"\"\"\n\u001b[1;32m-> 1050\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSCREENSHOT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1051\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1052\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mset_window_size\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwidth\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwindowHandle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'current'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    310\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    311\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 312\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    313\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    314\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mD:\\Users\\kevin\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mWebDriverException\u001b[0m: Message: Failed to decode response from marionette\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "import codecs\n",
    "import time\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "\n",
    "## ITS WORKING BRO\n",
    "\n",
    "\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(\"https://job.jobnet.dk/CV\")\n",
    "\n",
    "Username = driver.find_element_by_id('JobnetUsername')\n",
    "Password = driver.find_element_by_id('JobnetPassword')\n",
    "\n",
    "Username.send_keys(\"kevinruder@ymail.com\")\n",
    "Password.send_keys(\"\")\n",
    "\n",
    "#driver.find_element_by_id(\"Login\").text\n",
    "Password.submit()\n",
    "\n",
    "\n",
    "#driver.get()\n",
    "print(driver.current_url)\n",
    "\n",
    "# Waits for page with job listing to load.\n",
    "\n",
    "try:\n",
    "    element = WebDriverWait(driver, 10).until(\n",
    "        EC.presence_of_element_located((By.LINK_TEXT, \"Find Job\"))\n",
    "    )\n",
    "    \n",
    "finally:\n",
    "    element.click()\n",
    "    \n",
    "# Waits for search field to load\n",
    "try:\n",
    "    SearchField =  WebDriverWait(driver, 30).until(\n",
    "        EC.visibility_of_element_located((By.ID, \"SearchFieldAngular\"))\n",
    "    )\n",
    "finally:\n",
    "    driver.save_screenshot('screen.png')\n",
    "    SearchField.clear()\n",
    "    SearchField.send_keys(\"Data\")\n",
    "\n",
    "    \n",
    "try:\n",
    "    Buttons = WebDriverWait(driver, 10).until(\n",
    "        EC.presence_of_all_elements_located((By.TAG_NAME, \"a\"))\n",
    "    )\n",
    "finally:\n",
    "    driver.save_screenshot('screen.png')\n",
    "    \n",
    "driver.get(driver.current_url+\"&SearchString=Data\")\n",
    "\n",
    "DocumentNr = 0\n",
    "\n",
    "for Offset in range(0,1000,20):\n",
    "    \n",
    "    currentPage = driver.current_url\n",
    "    next_page = currentPage.replace(str(Offset),str(int(Offset)+20))\n",
    "    \n",
    "    try:\n",
    "        JobListings = WebDriverWait(driver, 10).until(\n",
    "        EC.presence_of_all_elements_located((By.TAG_NAME, \"a\"))\n",
    "    )\n",
    "        \n",
    "    finally:\n",
    "        time.sleep(20)\n",
    "        soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "        links = soup.find_all('a')\n",
    "        for s in links:\n",
    "            if str(s.get(\"href\")).find(\"Details\") >= 0:\n",
    "                time.sleep(15)\n",
    "                driver.get((s.get(\"href\")))\n",
    "                time.sleep(10)\n",
    "                soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "                print(\"here'some text:\")\n",
    "                mainBody = soup.find_all(\"div\",{'data-ng-if': 'details'})\n",
    "                newMainBody = BeautifulSoup(str(mainBody),'html.parser').get_text()\n",
    "                \n",
    "                # SAVE IT TO A FILE\n",
    "                text_file = codecs.open(\"Data_2/Document_\"+str(DocumentNr)+\".txt\", \"w\",encoding=\"utf-8\")\n",
    "                text_file.write(str(newMainBody))\n",
    "                text_file.close()\n",
    "                DocumentNr += 1\n",
    "\n",
    "        driver.get(next_page)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'driver' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-ad0c9a213c39>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'driver' is not defined"
     ]
    }
   ],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Time for Topic Modelling\n",
    "## importing gensim\n",
    "\n",
    "from gensim import corpora,models, similarities\n",
    "from collections import defaultdict\n",
    "import codecs\n",
    "\n",
    "## Library to help define stop words\n",
    "\n",
    "from stop_words import get_stop_words\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Defining some variables\n",
    "\n",
    "stop_words = get_stop_words('english')\n",
    "document_path = \"C:/Users/kevin/Data_2/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_tfidf_index(corpus_tfidf):\n",
    "\n",
    "    #extracts all indexes/words with a TFIDF of above 0.5\n",
    "\n",
    "    indexes = []\n",
    "\n",
    "    for doc in corpus_tfidf:\n",
    "        for a in doc:\n",
    "            #0.5 is the TFIDF THRESHOLD AND IS HARDCODED ATM, SHOULD BE A VARIABLE THAT IS DECLARED ON TOP of code\n",
    "            if (a[1] > 0.5):\n",
    "                indexes.append(a[0])\n",
    "\n",
    "    return indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_txt_from_doc():\n",
    "    \n",
    "    List_of_docs = []\n",
    "    \n",
    "    for x in range (0,800):\n",
    "        document = codecs.open(document_path+\"Document_\"+str(x)+\".txt\",\"r\",encoding=\"utf-8\")\n",
    "        List_of_docs.append(document.readlines())\n",
    "        \n",
    "    \n",
    "    return List_of_docs\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "List_of_documents = extract_txt_from_doc()\n",
    "\n",
    "for document in List_of_documents:\n",
    "    print(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}