maharshatra(ballarpur) fetching by name.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "\n",
    "from selenium import webdriver\n",
    "import time\n",
    "from selenium.webdriver.support.ui import Select\n",
    "import urllib\n",
    "import requests\n",
    "from selenium.webdriver import ActionChains\n",
    "import PIL\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import pytesseract\n",
    "import base64\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser = webdriver.Firefox(executable_path=r'/usr/bin/geckodriver')\n",
    "browser.get('http://103.23.150.139/marathi/')\n",
    "#...\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# elm1 =browser.find_element_by_id('ctl00_ContentPlaceHolder1_RadioButton1')\n",
    "# drp1 = Select(elm1)\n",
    "# drp1.select_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "elm1 =browser.find_element_by_id('ctl00_ContentPlaceHolder1_RadioButton1')\n",
    "elm1.click()\n",
    "time.sleep(2)\n",
    "elm2 =browser.find_element_by_id('ctl00_ContentPlaceHolder1_RadioButton4')\n",
    "elm2.click()\n",
    "time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ctl00_ContentPlaceHolder1_Drop2\n",
    "elm3 =browser.find_element_by_id('ctl00_ContentPlaceHolder1_Drop2')\n",
    "drp1 = Select(elm3)\n",
    "drp1.select_by_index(13)\n",
    "\n",
    "time.sleep(2)\n",
    "elm4 =browser.find_element_by_id('ctl00_ContentPlaceHolder1_Drop3')\n",
    "drp2 = Select(elm4)\n",
    "drp2.select_by_index(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'l'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# alpha = map(chr, range(97, 123)) \n",
    "# alpha\n",
    "import string\n",
    "alpha = string.ascii_lowercase\n",
    "# print(string.ascii_lowercase)\n",
    "# alpha[0]\n",
    "# # inputElement.clear()\n",
    "# inputElement = browser.find_element_by_xpath('//*[@id=\"ctl00_ContentPlaceHolder1_TextBox1\"]')\n",
    "# inputElement.clear()\n",
    "# # inputElement.send_keys(Keys.ENTER)\n",
    "# inputElement.send_keys(alpha[2])\n",
    "\n",
    "# alpha[18:]\n",
    "# hindiC = 'aiāīūṝṛ'\n",
    "# len(hindiC)\n",
    "# alpha = hindiC\n",
    "len(alpha)\n",
    "alpha[11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l\n"
     ]
    }
   ],
   "source": [
    "j = 11\n",
    "while j < 12:\n",
    "    \n",
    "    # for filling the first name from a-z\n",
    "    try:\n",
    "        time.sleep(5)\n",
    "\n",
    "        inputElement = browser.find_element_by_xpath('//*[@id=\"ctl00_ContentPlaceHolder1_TextBox3\"]')\n",
    "        inputElement.clear()\n",
    "        # inputElement.send_keys(Keys.ENTER)\n",
    "        inputElement.send_keys(alpha[j])\n",
    "\n",
    "        # for identifying captcha and entering the value\n",
    "\n",
    "        time.sleep(2)\n",
    "        img = browser.find_element_by_xpath('/html/body/form/div[3]/div[3]/div[3]/table/tbody/tr[6]/td[1]/img')\n",
    "\n",
    "\n",
    "        img_captcha_base64 = browser.execute_async_script(\"\"\"\n",
    "            var ele = arguments[0], callback = arguments[1];\n",
    "            ele.addEventListener('load', function fn(){\n",
    "              ele.removeEventListener('load', fn, false);\n",
    "              var cnv = document.createElement('canvas');\n",
    "              cnv.width = this.width; cnv.height = this.height;\n",
    "              cnv.getContext('2d').drawImage(this, 0, 0);\n",
    "              callback(cnv.toDataURL('image/jpeg').substring(22));\n",
    "            }, false);\n",
    "            ele.dispatchEvent(new Event('load'));\n",
    "            \"\"\", img)\n",
    "\n",
    "        # save the captcha to a file\n",
    "        with open(r\"captcha.jpg\", 'wb') as f:\n",
    "            f.write(base64.b64decode(img_captcha_base64))\n",
    "\n",
    "\n",
    "        img = PIL.Image.open('captcha.jpg')\n",
    "        data = pytesseract.image_to_string(img, config='-c tessedit_char_whitelist=+*-/0123456789= --psm 6')\n",
    "        result = int(data.split('+')[0]) + int(data[:-1].split('+')[1].replace(\" \", ''))\n",
    "\n",
    "        inputElement = browser.find_element_by_id('ctl00_ContentPlaceHolder1_Txtcaptcha2')\n",
    "        inputElement.clear()\n",
    "        inputElement.send_keys(str(result))\n",
    "\n",
    "        # clicking the search button\n",
    "        time.sleep(3)\n",
    "        search = browser.find_element_by_id('ctl00_ContentPlaceHolder1_Button2')\n",
    "        search.click()\n",
    "\n",
    "        # finding all items of table\n",
    "\n",
    "        time.sleep(20)\n",
    "\n",
    "    #     waitForLoad(/html/body/form/div[3]/div[3]/div[4]/center/div/table/tbody/tr[1]/td[1])\n",
    "        l = []\n",
    "        html = browser.page_source\n",
    "        soup = BeautifulSoup(html)\n",
    "\n",
    "        # print \"Finding...\"\n",
    "\n",
    "        for tag in soup.find_all('td'):\n",
    "        #     print(tag.text)\n",
    "            l.append(tag.text)\n",
    "\n",
    "        # for appending all data to excel sheet\n",
    "\n",
    "        ac = []\n",
    "        pn = []\n",
    "        sn = []\n",
    "        name = []\n",
    "        eid = []\n",
    "        gen = []\n",
    "        age = []\n",
    "        i=0\n",
    "        ab = l[14:]\n",
    "        while i < len(ab):\n",
    "            ac.append(ab[i])\n",
    "            i+=1\n",
    "            pn.append(ab[i])\n",
    "            i+=1\n",
    "            sn.append(ab[i])\n",
    "            i+=1\n",
    "            name.append(ab[i])\n",
    "            i+=1\n",
    "            eid.append(ab[i])\n",
    "            i+=1\n",
    "            gen.append(ab[i])\n",
    "            i+=1\n",
    "            age.append(ab[i])\n",
    "            i+=3\n",
    "        #     blank1.append(ab[i])\n",
    "        #     i=i+1\n",
    "        df = pd.DataFrame(np.column_stack([ac, pn, sn, name, eid, gen, age]), \n",
    "                               columns=['Assembly Constituency','Part No', 'Serial No', 'Name', 'Epic ID','Geder','Age'])\n",
    "        df.to_excel(\"/f_drive/Pdf of Constituencies/m\"+alpha[j]+\".xlsx\",index = False)\n",
    "        print(alpha[j])\n",
    "    except:\n",
    "        continue\n",
    "    j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def waitForLoad(inputXPath): \n",
    "    Wait = WebDriverWait(browser, PATIENCE_TIME)       \n",
    "    Wait.until(EC.presence_of_element_located((By.XPATH, inputXPath)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('/f_drive/Pdf of Constituencies/maharashtra (ballarpur)/'+alpha[j]+\".xlsx\")\n",
    "# df.head\n",
    "# df[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ac[0]\n",
    "a = []\n",
    "a.extend(df.iloc[:, 3])\n",
    "# a[-10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ac = []\n",
    "pn = []\n",
    "sn = []\n",
    "name = []\n",
    "eid = []\n",
    "gen = []\n",
    "age = []\n",
    "for i in range(len(alpha)):\n",
    "    try:\n",
    "        df = pd.read_excel('/f_drive/Pdf of Constituencies/l'+alpha[i]+\".xlsx\")\n",
    "        ac.extend(df.iloc[:,0])\n",
    "        pn.extend(df.iloc[:,1])\n",
    "        sn.extend(df.iloc[:,2])\n",
    "        name.extend(df.iloc[:,3])\n",
    "        eid.extend(df.iloc[:,4])\n",
    "        gen.extend(df.iloc[:,5])\n",
    "        age.extend(df.iloc[:,6])\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.column_stack([ac, pn, sn, name, eid, gen, age]), \n",
    "                           columns=['Assembly Constituency','Part No', 'Serial No', 'Name', 'Epic ID','Geder','Age'])\n",
    "df.to_excel(\"/f_drive/Pdf of Constituencies/maharashtra (ballarpur)/l\"+\"Maharashtra(ballarpur).xlsx\",index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf-8' codec can't decode byte 0x8f in position 22: invalid start byte",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-fd8e83b31b44>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mmore_itertools\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0munique_everseen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/f_drive/Pdf of Constituencies/maharashtra (ballarpur)/Maharashtra(ballarpur).xlsx'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'2.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mout_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mout_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwritelines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_everseen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/more_itertools/recipes.py\u001b[0m in \u001b[0;36munique_everseen\u001b[0;34m(iterable, key)\u001b[0m\n\u001b[1;32m    376\u001b[0m     \u001b[0mseenlist_add\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseenlist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 378\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0melement\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    379\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0melement\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mseenset\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/codecs.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, input, final)\u001b[0m\n\u001b[1;32m    320\u001b[0m         \u001b[0;31m# decode input (taking the buffer into account)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 322\u001b[0;31m         \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconsumed\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    323\u001b[0m         \u001b[0;31m# keep undecoded input until the next call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    324\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mconsumed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0x8f in position 22: invalid start byte"
     ]
    }
   ],
   "source": [
    "from more_itertools import unique_everseen\n",
    "with open('/f_drive/Pdf of Constituencies/maharashtra (ballarpur)/Maharashtra(ballarpur).xlsx','r') as f, open('2.csv','w') as out_file:\n",
    "    out_file.writelines(unique_everseen(f))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}