WebScrapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as BS\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start with a sample page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('/Users/adityakumar/Desktop/Machine Learning/index.html')\n",
    "html =f.read()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      "<head>\n",
      "<title>My Page</title>\n",
      "</head>\n",
      "<body bgcolor=\"yellow\">\n",
      "<h1>Welcome to my page!</h1>\n",
      "<br/><hr/>\n",
      "<p>Lorem Ipsum Lorem Ipsum Lorem Ipsum Lorem Ipsum</p>\n",
      "</body>\n",
      "</html>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adityakumar/Library/Python/2.7/lib/python/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 162 of the file /System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "soup = BS(html)\n",
    "print soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My Page\n",
      "My Page\n"
     ]
    }
   ],
   "source": [
    "print soup.html.head.title.text\n",
    "print soup.find_all('title')[0].text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scrape XKCD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "url ='https://xkcd.com/1530/'\n",
    "r= requests.get(url)\n",
    "print r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      "<head>\n",
      "<link rel=\"stylesheet\" type=\"text/css\" href=\"/s/b0dcca.css\" title=\"Default\"/>\n",
      "<title>xkcd: Keyboard Mash</title>\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\"/>\n",
      "<link rel=\"shortcut icon\" href=\"/s/919f27.ico\" type=\"image/x-icon\"/>\n",
      "<link rel=\"icon\" href=\"/s/919f27.ico\" type=\"image/x-icon\"/>\n",
      "<link rel=\"alternate\" type=\"application/atom+xml\" title=\"Atom 1.0\" href=\"/atom.xml\"/>\n",
      "<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS 2.0\" href=\"/rss.xml\"/>\n",
      "<script type=\"text/javascript\" src=\"/s/b66ed7.js\" async></script>\n",
      "<script type=\"text/javascript\" src=\"/s/1b9456.js\" async></script>\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"topContainer\">\n",
      "<div id=\"topLeft\">\n",
      "<ul>\n",
      "<li><a href=\"/archive\">Archive</a></li>\n",
      "<li><a href=\"http://what-if.xkcd.com\">What If?</a></li>\n",
      "<li><a href=\"http://blag.xkcd.com\">Blag</a></li>\n",
      "<li><a href=\"http://store.xkcd.com/\">Store</a></li>\n",
      "<li><a rel=\"author\" href=\"/about\">About</a></li>\n",
      "</ul>\n",
      "</div>\n",
      "<div id=\"topRight\">\n",
      "<div id=\"masthead\">\n",
      "<span><a href=\"/\"><img src=\"/s/0b7742.png\" alt=\"xkcd.com logo\" height=\"83\" width=\"185\"/></a></span>\n",
      "<span id=\"slogan\">A webcomic of romance,<br/> sarcasm, math, and language.</span>\n",
      "</div>\n",
      "<div id=\"news\">\n",
      "xkcd updates every Monday, Wednesday, and Friday.\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "<div id=\"bgLeft\" class=\"bg box\"></div>\n",
      "<div id=\"bgRight\" class=\"bg box\"></div>\n",
      "</div>\n",
      "<div id=\"middleContainer\" class=\"box\">\n",
      "\n",
      "<div id=\"ctitle\">Keyboard Mash</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a rel=\"prev\" href=\"/1529/\" accesskey=\"p\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a rel=\"next\" href=\"/1531/\" accesskey=\"n\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<div id=\"comic\">\n",
      "<img src=\"//imgs.xkcd.com/comics/keyboard_mash.png\" title=\"WHY DON&#39;T YOU COME HANG OUT INSIDE MY HOUSE. WE CAN COOK BREAD AND CHAT ABOUT OUR INTERNAL SKELETONS.\" alt=\"Keyboard Mash\" srcset=\"//imgs.xkcd.com/comics/keyboard_mash_2x.png 2x\"/>\n",
      "</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a rel=\"prev\" href=\"/1529/\" accesskey=\"p\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a rel=\"next\" href=\"/1531/\" accesskey=\"n\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<br />\n",
      "Permanent link to this comic: https://xkcd.com/1530/<br />\n",
      "Image URL (for hotlinking/embedding): https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "<div id=\"transcript\" style=\"display: none\">[[A person walks toward their computer desk as the screen indicates &quot;new chat message.&quot;]]\n",
      "\n",
      "[[The screen shows a text conversation (where lines are contained in speech bubble boxes) between White Hat Guy and the first person.]]\n",
      "\n",
      "WHG: Can&#39;t sleep. Stupid dogs keep barking.\n",
      "          FJAFJKLDWSKF7JKFDJ\n",
      "\n",
      "FP: Ugh, I&#39;m sorry. Maybe you could...\n",
      "      ...OK, wait. I have to ask. \n",
      "      How did you hit a &quot;7&quot; in the middle there?\n",
      "\n",
      "WHG: Huh?\n",
      "          I was just randomly keyboard mashing.\n",
      "\n",
      "FP: Sorry, right.\n",
      "\n",
      "WHG: Anyway,\n",
      "\n",
      "FP: ...I know this is silly, but like... all your hands were clearly on the home row.\n",
      "     I don&#39;t get how one finger could have stretched up to the &quot;7.&quot;\n",
      "\n",
      "WHG: Why do you always fixate on these bizarre details?\n",
      "\n",
      "FG: I don&#39;t know.\n",
      "      Sorry.\n",
      "\n",
      "WHG: It&#39;s weird, is all.\n",
      "         \n",
      "[[A giant spider dangles from the ceiling, typing at a laptop on a desk, while White Hat Guy is suspended upside down, wrapped in spider thread. The chair lies on the floor on its side. The conversation continues on the computer.]]\n",
      "Spider:I am a normal human typing with my human hands.\n",
      "FP: Yeah, of course. \n",
      "      I know.\n",
      "((MMM!! MMPH!!!))\n",
      " \n",
      "{{Title text: WHY DON&#39;T YOU COME HANG OUT INSIDE MY HOUSE. WE CAN COOK BREAD AND CHAT ABOUT OUR INTERNAL SKELETONS.}}</div>\n",
      "</div>\n",
      "<div id=\"bottom\" class=\"box\">\n",
      "<img src=\"//imgs.xkcd.com/s/a899e84.jpg\" width=\"520\" height=\"100\" alt=\"Selected Comics\" usemap=\"#comicmap\"/>\n",
      "<map id=\"comicmap\" name=\"comicmap\">\n",
      "<area shape=\"rect\" coords=\"0,0,100,100\" href=\"/150/\" alt=\"Grownups\"/>\n",
      "<area shape=\"rect\" coords=\"104,0,204,100\" href=\"/730/\" alt=\"Circuit Diagram\"/>\n",
      "<area shape=\"rect\" coords=\"208,0,308,100\" href=\"/162/\" alt=\"Angular Momentum\"/>\n",
      "<area shape=\"rect\" coords=\"312,0,412,100\" href=\"/688/\" alt=\"Self-Description\"/>\n",
      "<area shape=\"rect\" coords=\"416,0,520,100\" href=\"/556/\" alt=\"Alternative Energy Revolution\"/>\n",
      "</map>\n",
      "<div>\n",
      "<!--\n",
      "Search comic titles and transcripts:\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/jsapi\"></script>\n",
      "<script type=\"text/javascript\">google.load('search', '1');google.setOnLoadCallback(function() {google.search.CustomSearchControl.attachAutoCompletion('012652707207066138651:zudjtuwe28q',document.getElementById('q'),'cse-search-box');});</script>\n",
      "<form action=\"//www.google.com/cse\" id=\"cse-search-box\">\n",
      "<div>\n",
      "<input type=\"hidden\" name=\"cx\" value=\"012652707207066138651:zudjtuwe28q\"/>\n",
      "<input type=\"hidden\" name=\"ie\" value=\"UTF-8\"/>\n",
      "<input type=\"text\" name=\"q\" id=\"q\" size=\"31\"/>\n",
      "<input type=\"submit\" name=\"sa\" value=\"Search\"/>\n",
      "</div>\n",
      "</form>\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/cse/brand?form=cse-search-box&amp;lang=en\"></script>\n",
      "-->\n",
      "<a href=\"/rss.xml\">RSS Feed</a> - <a href=\"/atom.xml\">Atom Feed</a>\n",
      "</div>\n",
      "<br />\n",
      "<div id=\"comicLinks\">\n",
      "Comics I enjoy:<br/>\n",
      "        <a href=\"http://threewordphrase.com/\">Three Word Phrase</a>,\n",
      "        <a href=\"http://www.smbc-comics.com/\">SMBC</a>,\n",
      "        <a href=\"http://www.qwantz.com\">Dinosaur Comics</a>,\n",
      "        <a href=\"http://oglaf.com/\">Oglaf</a> (nsfw),\n",
      "        <a href=\"http://www.asofterworld.com\">A Softer World</a>,\n",
      "        <a href=\"http://buttersafe.com/\">Buttersafe</a>,\n",
      "        <a href=\"http://pbfcomics.com/\">Perry Bible Fellowship</a>,\n",
      "        <a href=\"http://questionablecontent.net/\">Questionable Content</a>,\n",
      "        <a href=\"http://www.buttercupfestival.com/\">Buttercup Festival</a>,\n",
      "        <a href=\"http://www.mspaintadventures.com/?s=6&p=001901\">Homestuck</a>,\n",
      "\t<a href=\"http://www.jspowerhour.com/\">Junior Scientist Power Hour</a>,\n",
      "</div>\n",
      "<br />\n",
      "<div id=\"comicLinks\">\n",
      "Other things:<br/>\n",
      "        <a href=\"http://womenalsoknowstuff.com/\">Women Also Know Stuff</a>,\n",
      "        <a href=\"https://techsolidarity.org/\">Tech Solidarity</a>,\n",
      "        <a href=\"https://medium.com/civic-tech-thoughts-from-joshdata/so-you-want-to-reform-democracy-7f3b1ef10597\">Tips on technology and government</a>\n",
      "</div>\n",
      "<br />\n",
      "<center>\n",
      "<div id=\"footnote\" style=\"width:70%\">xkcd.com is best viewed with Netscape Navigator 4.0 or below on a Pentium 3&plusmn;1 emulated in Javascript on an Apple IIGS<br />at a screen resolution of 1024x1. Please enable your ad blockers, disable high-heat drying, and remove your device<br />from Airplane Mode and set it to Boat Mode. For security reasons, please leave caps lock on while browsing.</div>\n",
      "</center>\n",
      "<div id=\"licenseText\">\n",
      "<p>\n",
      "This work is licensed under a\n",
      "<a href=\"http://creativecommons.org/licenses/by-nc/2.5/\">Creative Commons Attribution-NonCommercial 2.5 License</a>.\n",
      "</p><p>\n",
      "This means you're free to copy and share these comics (but not to sell them). <a rel=\"license\" href=\"/license.html\">More details</a>.</p>\n",
      "</div>\n",
      "</div>\n",
      "</body>\n",
      "<!-- Layout by Ian Clasbey, davean, and chromakode -->\n",
      "</html>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print r.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BS(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://imgs.xkcd.com/comics/keyboard_mash.png\n"
     ]
    }
   ],
   "source": [
    "divs=soup.find_all('div', id='comic')\n",
    "image_div= divs[0]\n",
    "image_url='https:' + image_div.img['src']\n",
    "print image_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#wrapping above all\n",
    "def get_xkcd_image(idx=1000):\n",
    "    url='https://xkcd.com/{}/'.format(idx)\n",
    "    \n",
    "    r=requests.get(url)\n",
    "    if r.status_code==200:\n",
    "        soup=BS(r.text)\n",
    "        img_url='https:' + soup.find_all('div', id='comic')[0].img['src']\n",
    "    return image_url\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "101 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "201 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "301 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "401 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "501 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "601 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "701 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "801 https://imgs.xkcd.com/comics/keyboard_mash.png\n",
      "901 https://imgs.xkcd.com/comics/keyboard_mash.png\n"
     ]
    }
   ],
   "source": [
    "for ix in range(1,1000,100):\n",
    "    print ix, get_xkcd_image(ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scrapping datatau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "url='http://www.datatau.com/'\n",
    "r=requests.get(url)\n",
    "print r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup =BS(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_table=soup.find_all('table')[2]\n",
    "rows=main_table.find_all('tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Breaking into Data Science in Seattle\n",
      "--------------------------------------------------\n",
      "https://blog.insightdatascience.com/seattle-insight-data-science-one-year-anniversary-9e51153d3779\n",
      "**************************************************\n",
      "A Scikit-learn pipeline in Wallaroo\n",
      "--------------------------------------------------\n",
      "https://blog.wallaroolabs.com/2018/02/a-scikit-learn-pipeline-in-wallaroo/\n",
      "**************************************************\n",
      "Defining data science in 2018\n",
      "--------------------------------------------------\n",
      "https://yanirseroussi.com/2018/07/22/defining-data-science-in-2018/\n",
      "**************************************************\n",
      "Pandas Tutorial 2: Aggregation and Grouping\n",
      "--------------------------------------------------\n",
      "https://data36.com/pandas-tutorial-2-aggregation-and-grouping/\n",
      "**************************************************\n",
      "Fast, Flexible, Easy and Intuitive: How to Speed Up Your Pandas Projects\n",
      "--------------------------------------------------\n",
      "https://realpython.com/fast-flexible-pandas/\n",
      "**************************************************\n",
      "Practical Apache Spark in 10 minutes. Part 4 — MLlib\n",
      "--------------------------------------------------\n",
      "https://medium.com/data-science-school/practical-apache-spark-in-10-minutes-part-4-mllib-fca02fecf5b8\n",
      "**************************************************\n",
      "The most in-demand tech roles and industries for 2018 [Data Visualized]\n",
      "--------------------------------------------------\n",
      "https://statsbot.co/blog/best-tech-roles-and-industries-in-2018\n",
      "**************************************************\n",
      "Multi data and outputs volumes for you DL and ML experiments with Polyaxon\n",
      "--------------------------------------------------\n",
      "https://medium.com/polyaxon/mounting-multiple-data-and-outputs-volumes-41f2d912f2f8\n",
      "**************************************************\n",
      "Top 10 Data Science Use Cases in Insurance\n",
      "--------------------------------------------------\n",
      "https://medium.com/activewizards-machine-learning-company/top-10-data-science-use-cases-in-insurance-8cade8a13ee1\n",
      "**************************************************\n",
      "Pandas Tutorial 1: Basics (Reading Data Files, DataFrames, Data Selection)\n",
      "--------------------------------------------------\n",
      "https://data36.com/pandas-tutorial-1-basics-reading-data-files-dataframes-data-selection/\n",
      "**************************************************\n",
      "Using GPUs to Analyze Mice in Space\n",
      "--------------------------------------------------\n",
      "https://www.mapd.com/blog/analyzing-nasa-spaceflight-microgravity-effects-on-mice-repertoire/\n",
      "**************************************************\n",
      "Reading and writing to Google Spreadsheets using Python\n",
      "--------------------------------------------------\n",
      "https://www.interviewqs.com/blog/pythonsheets\n",
      "**************************************************\n",
      "A great guide to K-Means Clustering - Machine Learning\n",
      "--------------------------------------------------\n",
      "http://byteacademy.co/blog/k-means-clustering/\n",
      "**************************************************\n",
      "Tesla — The Amazon and Apple of the Self Driving Future\n",
      "--------------------------------------------------\n",
      "https://medium.com/acing-ai/tesla-the-amazon-and-apple-of-the-self-driving-future-ad37e12922f3\n",
      "**************************************************\n",
      "17 best python libraries\n",
      "--------------------------------------------------\n",
      "https://boostlog.io/@nixus89896/17-best-python-libraries-5ad83a3247018500491f3e5b\n",
      "**************************************************\n",
      "Hackers Guide to Healthcare Data\n",
      "--------------------------------------------------\n",
      "http://rohtas.space/2018-05-25-hackers-guide-to-healthcare-data\n",
      "**************************************************\n",
      "Top 20 Python libraries for data science in 2018\n",
      "--------------------------------------------------\n",
      "https://medium.com/activewizards-machine-learning-company/top-20-python-libraries-for-data-science-in-2018-2ae7d1db8049\n",
      "**************************************************\n",
      "Top 8 Sites To Make Money By Uploading Files\n",
      "--------------------------------------------------\n",
      "https://tech-trending.com/top-8-sites-to-make-money-by-uploading-files/\n",
      "**************************************************\n",
      "Hypothesis Testing I: Prerequisites\n",
      "--------------------------------------------------\n",
      "http://ajknblog.com/basics/hypothesis-testing-prerequisites/\n",
      "**************************************************\n",
      "How to choose the right analytics platform for your team\n",
      "--------------------------------------------------\n",
      "https://www.gethighlights.co/blog/choose-right-analytics-platform/\n",
      "**************************************************\n",
      "Comparison of top data science libraries for Python, R and Scala [Infographic]\n",
      "--------------------------------------------------\n",
      "https://medium.com/activewizards-machine-learning-company/comparison-of-top-data-science-libraries-for-python-r-and-scala-infographic-574069949267\n",
      "**************************************************\n",
      "How I built a Self Flying Drone to track People in under 50 lines of code\n",
      "--------------------------------------------------\n",
      "https://medium.com/nanonets/how-i-built-a-self-flying-drone-to-track-people-in-under-50-lines-of-code-7485de7f828e\n",
      "**************************************************\n",
      "50 Most Popular Python Projects in 2018\n",
      "--------------------------------------------------\n",
      "https://boostlog.io/@bily809/50-most-popular-python-projects-in-2018-5aea8e1c47018500491f4361\n",
      "**************************************************\n",
      "Gefilter Fish: Finding concise topics from Amazon’s customer reviews\n",
      "--------------------------------------------------\n",
      "https://blog.insightdatascience.com/gefilter-fish-finding-concise-topics-from-amazons-customer-reviews-dd14d673ed6d\n",
      "**************************************************\n",
      "SQL filter query on FIFA player data\n",
      "--------------------------------------------------\n",
      "https://statsbot.co/blog/sql-filter-query/\n",
      "**************************************************\n",
      "Monitoring DevOps: Where are we now? [Infographic]\n",
      "--------------------------------------------------\n",
      "https://www.circonus.com/2018/07/monitoring-devops-where-are-we-now-infographic/\n",
      "**************************************************\n",
      "Twitter Data Science Interview Questions - Acing the AI Interview\n",
      "--------------------------------------------------\n",
      "https://medium.com/acing-ai/twitter-data-science-interview-questions-acing-the-ai-interview-f8204a97159f\n",
      "**************************************************\n",
      "The unreasonable effectiveness of Deep Learning Representations\n",
      "--------------------------------------------------\n",
      "https://blog.insightdatascience.com/the-unreasonable-effectiveness-of-deep-learning-representations-4ce83fc663cf\n",
      "**************************************************\n",
      "SLO’s & You: A Guide To Service Level Objectives\n",
      "--------------------------------------------------\n",
      "https://www.circonus.com/2018/07/a-guide-to-service-level-objectives/\n",
      "**************************************************\n",
      "ElastiK Nearest Neighbors: LSH and Elasticsearch for Scalable Online KNN Search\n",
      "--------------------------------------------------\n",
      "https://blog.insightdatascience.com/elastik-nearest-neighbors-4b1f6821bd62\n",
      "**************************************************\n"
     ]
    }
   ],
   "source": [
    "for ix in range(0, 90, 3):\n",
    "    td_elem= rows[ix].find_all('td')[-1]\n",
    "    print td_elem.a.text\n",
    "    print '-'*50\n",
    "    print td_elem.a['href']\n",
    "    print '*'*50\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "wikipedia scrapping\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "url='https://en.wikipedia.org/wiki/List_of_countries_by_national_capital,_largest_and_second-largest_cities'\n",
    "r= requests.get(url)\n",
    "print r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=BS(r.text)\n",
    "#print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "con_cap=pd.DataFrame([], columns=['Country', 'Capital'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_table=data.find_all(\"table\", {\"class\":\"wikitable sortable\"})[0]\n",
    "rows=main_table.find_all('tbody')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Abkhazia\n",
      "Afghanistan\n",
      "Akrotiri and Dhekelia\n",
      "Åland Islands\n",
      "Albania\n",
      "Algeria\n",
      "American Samoa\n",
      "Andorra\n",
      "Angola\n",
      "Anguilla\n",
      "Antigua and Barbuda\n",
      "Argentina\n",
      "Armenia\n",
      "Republic of Artsakh\n",
      "Aruba\n",
      "Australia\n",
      "Austria\n",
      "Azerbaijan\n",
      "The Bahamas\n",
      "Bahrain\n",
      "Bangladesh\n",
      "Barbados\n",
      "Belarus\n",
      "Belgium\n",
      "Belize\n",
      "Benin\n",
      "Bermuda\n",
      "Bhutan\n",
      "Bolivia\n",
      "Bosnia and Herzegovina\n",
      "Botswana\n",
      "Brazil\n",
      "Brunei\n",
      "Bulgaria\n",
      "Burkina Faso\n",
      "Burundi\n",
      "Cambodia\n",
      "Cameroon\n",
      "Canada\n",
      "Cape Verde\n",
      "Cayman Islands\n",
      "Central African Republic\n",
      "Chad\n",
      "Chile\n",
      "China\n",
      "Christmas Island\n",
      "Cocos (Keeling) Islands\n",
      "Colombia\n",
      "Comoros\n",
      "Democratic Republic of the Congo\n",
      "Republic of the Congo\n",
      "Cook Islands\n",
      "Costa Rica\n",
      "Ivory Coast\n",
      "Croatia\n",
      "Cuba\n",
      "Curaçao\n",
      "Cyprus\n",
      "Czech Republic\n",
      "Denmark\n",
      "Djibouti\n",
      "Dominica\n",
      "Dominican Republic\n",
      "East Timor\n",
      "Ecuador\n",
      "Egypt\n",
      "El Salvador\n",
      "Equatorial Guinea\n",
      "Eritrea\n",
      "Estonia\n",
      "Ethiopia\n",
      "Falkland Islands\n",
      "Faroe Islands\n",
      "Fiji\n",
      "Finland\n",
      "France\n",
      "French Guiana\n",
      "French Polynesia\n",
      "Gabon\n",
      "The Gambia\n",
      "Georgia (country)\n",
      "Germany\n",
      "Rhine-Ruhr\n",
      "Ghana\n",
      "Gibraltar\n",
      "Greece\n",
      "Greenland\n",
      "Grenada\n",
      "Guam\n",
      "Guatemala\n",
      "Guernsey\n",
      "Guinea\n",
      "Guinea-Bissau\n",
      "Guyana\n",
      "Haiti\n",
      "Honduras\n",
      "Hungary\n",
      "Iceland\n",
      "India\n",
      "Indonesia\n",
      "Iran\n",
      "Iraq\n",
      "Republic of Ireland\n",
      "Isle of Man\n",
      "Israel\n",
      "Italy\n",
      "Milan\n",
      "Jamaica\n",
      "Japan\n",
      "Jersey\n",
      "Jordan\n",
      "Kazakhstan\n",
      "Kenya\n",
      "Kiribati\n",
      "North Korea\n",
      "South Korea\n",
      "Kosovo\n",
      "Kuwait\n",
      "Kyrgyzstan\n",
      "Laos\n",
      "Latvia\n",
      "Lebanon\n",
      "Lesotho\n",
      "Liberia\n",
      "Libya\n",
      "Liechtenstein\n",
      "Lithuania\n",
      "Luxembourg\n",
      "Republic of Macedonia\n",
      "Madagascar\n",
      "Malawi\n",
      "Malaysia\n",
      "Maldives\n",
      "Mali\n",
      "Malta\n",
      "Marshall Islands\n",
      "Mauritania\n",
      "Mauritius\n",
      "Mayotte\n",
      "Mexico\n",
      "Federated States of Micronesia\n",
      "Moldova\n",
      "Monaco\n",
      "Mongolia\n",
      "Montenegro\n",
      "Montserrat\n",
      "Morocco\n",
      "Mozambique\n",
      "Myanmar\n",
      "Namibia\n",
      "Nauru\n",
      "Nepal\n",
      "Netherlands\n",
      "New Caledonia\n",
      "New Zealand\n",
      "Nicaragua\n",
      "Niger\n",
      "Nigeria\n",
      "Niue\n",
      "Norfolk Island\n",
      "Northern Cyprus\n",
      "Northern Mariana Islands\n",
      "Norway\n",
      "Oman\n",
      "Pakistan\n",
      "Palau\n",
      "State of Palestine\n",
      "Panama\n",
      "Papua New Guinea\n",
      "Paraguay\n",
      "Peru\n",
      "Philippines\n",
      "Manila\n",
      "Pitcairn Islands\n",
      "Poland\n",
      "Portugal\n",
      "Puerto Rico\n",
      "Qatar\n",
      "Romania\n",
      "Russia\n",
      "Rwanda\n",
      "Saint Barthélemy\n",
      "Saint Helena\n",
      "Saint Kitts and Nevis\n",
      "Saint Lucia\n",
      "Collectivity of Saint Martin\n",
      "Saint Pierre and Miquelon\n",
      "Saint Vincent and the Grenadines\n",
      "Samoa\n",
      "San Marino\n",
      "São Tomé and Príncipe\n",
      "Saudi Arabia\n",
      "Senegal\n",
      "Serbia\n",
      "Seychelles\n",
      "Sierra Leone\n",
      "Singapore\n",
      "Sint Maarten\n",
      "Slovakia\n",
      "Slovenia\n",
      "Solomon Islands\n",
      "Somalia\n",
      "Somaliland\n",
      "South Africa\n",
      "Johannesburg\n",
      "South Georgia and the South Sandwich Islands\n",
      "South Ossetia\n",
      "South Sudan\n",
      "Spain\n",
      "Sri Lanka\n",
      "Sudan\n",
      "Khartoum\n",
      "Suriname\n",
      "Svalbard\n",
      "Swaziland\n",
      "Sweden\n",
      "Switzerland\n",
      "Syria\n",
      "Taiwan\n",
      "Tajikistan\n",
      "Tanzania\n",
      "Thailand\n",
      "Togo\n",
      "Tokelau\n",
      "Tonga\n",
      "Transnistria\n",
      "Trinidad and Tobago\n",
      "Tunisia\n",
      "Turkey\n",
      "Turkmenistan\n",
      "Turks and Caicos Islands\n",
      "Tuvalu\n",
      "Uganda\n",
      "Ukraine\n",
      "United Arab Emirates\n",
      "United Kingdom\n",
      "United States\n",
      "Uruguay\n",
      "Uzbekistan\n",
      "Vanuatu\n",
      "Vatican City\n",
      "Venezuela\n",
      "Vietnam\n",
      "British Virgin Islands\n",
      "United States Virgin Islands\n",
      "Wallis and Futuna\n",
      "Western Sahara\n",
      "Yemen\n",
      "Zambia\n",
      "Zimbabwe\n"
     ]
    }
   ],
   "source": [
    "#selenium for dynamic pages\n",
    "\n",
    "for ix in range(4, 265):\n",
    "    col=rows.find_all('tr')[ix]\n",
    "    coun=col.find_all('td')[0]    \n",
    "    try:\n",
    "        cap=col.find_all('td')[1]\n",
    "    except IndexError:\n",
    "        continue\n",
    "    country=coun.a['title']\n",
    "    try:\n",
    "        capital=cap.a['title']\n",
    "    except TypeError:\n",
    "        capiatl=None\n",
    "    print country    \n",
    "    data={\n",
    "        'Country' : country,\n",
    "        'Capital' : capital\n",
    "    }\n",
    "    \n",
    "    con_cap=con_cap.append(data, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Capital</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abkhazia</td>\n",
       "      <td>Sukhumi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Kabul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Akrotiri and Dhekelia</td>\n",
       "      <td>Episkopi Cantonment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Finland</td>\n",
       "      <td>Mariehamn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Albania</td>\n",
       "      <td>Tirana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Algeria</td>\n",
       "      <td>Algiers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>American Samoa</td>\n",
       "      <td>Pago Pago</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Andorra</td>\n",
       "      <td>Andorra la Vella</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Angola</td>\n",
       "      <td>Luanda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Anguilla</td>\n",
       "      <td>The Valley, Anguilla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>St. John's, Antigua and Barbuda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Argentina</td>\n",
       "      <td>Buenos Aires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Armenia</td>\n",
       "      <td>Yerevan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Republic of Artsakh</td>\n",
       "      <td>Stepanakert</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Aruba</td>\n",
       "      <td>Oranjestad, Aruba</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Canberra</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Austria</td>\n",
       "      <td>Vienna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Baku</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The Bahamas</td>\n",
       "      <td>Nassau, Bahamas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Bahrain</td>\n",
       "      <td>Manama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>Dhaka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Barbados</td>\n",
       "      <td>Bridgetown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Belarus</td>\n",
       "      <td>Minsk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Belgium</td>\n",
       "      <td>Brussels</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Belize</td>\n",
       "      <td>Belmopan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Benin</td>\n",
       "      <td>Porto-Novo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Bermuda</td>\n",
       "      <td>Hamilton, Bermuda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Bhutan</td>\n",
       "      <td>Thimphu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Bolivia</td>\n",
       "      <td>Sucre</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Bosnia and Herzegovina</td>\n",
       "      <td>Sarajevo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>Tanzania</td>\n",
       "      <td>Dodoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>Thailand</td>\n",
       "      <td>Bangkok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>Togo</td>\n",
       "      <td>Lomé</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>Tokelau</td>\n",
       "      <td>Lomé</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>Tonga</td>\n",
       "      <td>Nukuʻalofa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>Transnistria</td>\n",
       "      <td>Tiraspol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>Trinidad and Tobago</td>\n",
       "      <td>Port of Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>Tunisia</td>\n",
       "      <td>Tunis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>Turkey</td>\n",
       "      <td>Ankara</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>Turkmenistan</td>\n",
       "      <td>Ashgabat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>Turks and Caicos Islands</td>\n",
       "      <td>Cockburn Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>Tuvalu</td>\n",
       "      <td>Funafuti</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>Uganda</td>\n",
       "      <td>Kampala</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>Kiev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>Abu Dhabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>London</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>United States</td>\n",
       "      <td>Washington, D.C.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Uruguay</td>\n",
       "      <td>Montevideo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Uzbekistan</td>\n",
       "      <td>Tashkent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>Vanuatu</td>\n",
       "      <td>Port Vila</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Vatican City</td>\n",
       "      <td>Vatican City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Venezuela</td>\n",
       "      <td>Caracas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Hanoi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>British Virgin Islands</td>\n",
       "      <td>Road Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>United States Virgin Islands</td>\n",
       "      <td>Charlotte Amalie, United States Virgin Islands</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Wallis and Futuna</td>\n",
       "      <td>Mata-Utu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>Western Sahara</td>\n",
       "      <td>Laayoune</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>Yemen</td>\n",
       "      <td>Sana'a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>Zambia</td>\n",
       "      <td>Lusaka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>Zimbabwe</td>\n",
       "      <td>Harare</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Country  \\\n",
       "0                        Abkhazia   \n",
       "1                     Afghanistan   \n",
       "2           Akrotiri and Dhekelia   \n",
       "3                         Finland   \n",
       "4                         Albania   \n",
       "5                         Algeria   \n",
       "6                  American Samoa   \n",
       "7                         Andorra   \n",
       "8                          Angola   \n",
       "9                        Anguilla   \n",
       "10            Antigua and Barbuda   \n",
       "11                      Argentina   \n",
       "12                        Armenia   \n",
       "13            Republic of Artsakh   \n",
       "14                          Aruba   \n",
       "15                      Australia   \n",
       "16                        Austria   \n",
       "17                     Azerbaijan   \n",
       "18                    The Bahamas   \n",
       "19                        Bahrain   \n",
       "20                     Bangladesh   \n",
       "21                       Barbados   \n",
       "22                        Belarus   \n",
       "23                        Belgium   \n",
       "24                         Belize   \n",
       "25                          Benin   \n",
       "26                        Bermuda   \n",
       "27                         Bhutan   \n",
       "28                        Bolivia   \n",
       "29         Bosnia and Herzegovina   \n",
       "..                            ...   \n",
       "220                      Tanzania   \n",
       "221                      Thailand   \n",
       "222                          Togo   \n",
       "223                       Tokelau   \n",
       "224                         Tonga   \n",
       "225                  Transnistria   \n",
       "226           Trinidad and Tobago   \n",
       "227                       Tunisia   \n",
       "228                        Turkey   \n",
       "229                  Turkmenistan   \n",
       "230      Turks and Caicos Islands   \n",
       "231                        Tuvalu   \n",
       "232                        Uganda   \n",
       "233                       Ukraine   \n",
       "234          United Arab Emirates   \n",
       "235                United Kingdom   \n",
       "236                 United States   \n",
       "237                       Uruguay   \n",
       "238                    Uzbekistan   \n",
       "239                       Vanuatu   \n",
       "240                  Vatican City   \n",
       "241                     Venezuela   \n",
       "242                       Vietnam   \n",
       "243        British Virgin Islands   \n",
       "244  United States Virgin Islands   \n",
       "245             Wallis and Futuna   \n",
       "246                Western Sahara   \n",
       "247                         Yemen   \n",
       "248                        Zambia   \n",
       "249                      Zimbabwe   \n",
       "\n",
       "                                            Capital  \n",
       "0                                           Sukhumi  \n",
       "1                                             Kabul  \n",
       "2                               Episkopi Cantonment  \n",
       "3                                         Mariehamn  \n",
       "4                                            Tirana  \n",
       "5                                           Algiers  \n",
       "6                                         Pago Pago  \n",
       "7                                  Andorra la Vella  \n",
       "8                                            Luanda  \n",
       "9                              The Valley, Anguilla  \n",
       "10                  St. John's, Antigua and Barbuda  \n",
       "11                                     Buenos Aires  \n",
       "12                                          Yerevan  \n",
       "13                                      Stepanakert  \n",
       "14                                Oranjestad, Aruba  \n",
       "15                                         Canberra  \n",
       "16                                           Vienna  \n",
       "17                                             Baku  \n",
       "18                                  Nassau, Bahamas  \n",
       "19                                           Manama  \n",
       "20                                            Dhaka  \n",
       "21                                       Bridgetown  \n",
       "22                                            Minsk  \n",
       "23                                         Brussels  \n",
       "24                                         Belmopan  \n",
       "25                                       Porto-Novo  \n",
       "26                                Hamilton, Bermuda  \n",
       "27                                          Thimphu  \n",
       "28                                            Sucre  \n",
       "29                                         Sarajevo  \n",
       "..                                              ...  \n",
       "220                                          Dodoma  \n",
       "221                                         Bangkok  \n",
       "222                                            Lomé  \n",
       "223                                            Lomé  \n",
       "224                                      Nukuʻalofa  \n",
       "225                                        Tiraspol  \n",
       "226                                   Port of Spain  \n",
       "227                                           Tunis  \n",
       "228                                          Ankara  \n",
       "229                                        Ashgabat  \n",
       "230                                   Cockburn Town  \n",
       "231                                        Funafuti  \n",
       "232                                         Kampala  \n",
       "233                                            Kiev  \n",
       "234                                       Abu Dhabi  \n",
       "235                                          London  \n",
       "236                                Washington, D.C.  \n",
       "237                                      Montevideo  \n",
       "238                                        Tashkent  \n",
       "239                                       Port Vila  \n",
       "240                                    Vatican City  \n",
       "241                                         Caracas  \n",
       "242                                           Hanoi  \n",
       "243                                       Road Town  \n",
       "244  Charlotte Amalie, United States Virgin Islands  \n",
       "245                                        Mata-Utu  \n",
       "246                                        Laayoune  \n",
       "247                                          Sana'a  \n",
       "248                                          Lusaka  \n",
       "249                                          Harare  \n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con_cap.head(260)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "con_cap.loc[3,'Country']='Finland'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_csv(con_cap,'/Users/adityakumar/Desktop/wiki1.csv', encoding='utf7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_csv?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Libraries for Web Scrapping\n",
    "#selenium #scrapy #MechanicalSoup #lxml #Requests #Beautiful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = pd.io.html.read_html(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Country or territory</td>\n",
       "      <td>Capital</td>\n",
       "      <td>City proper</td>\n",
       "      <td>Source</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Largest</td>\n",
       "      <td>Second largest</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Metropolitan area (if different)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Largest</td>\n",
       "      <td>Second largest</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abkhazia (Georgia)</td>\n",
       "      <td>Sukhumi</td>\n",
       "      <td>Gagra</td>\n",
       "      <td>[1]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>Kabul</td>\n",
       "      <td>Kandahar</td>\n",
       "      <td>[citation needed]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Akrotiri and Dhekelia (UK)</td>\n",
       "      <td>Episkopi Cantonment</td>\n",
       "      <td>Ayios Nikolaos</td>\n",
       "      <td>[citation needed]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Åland Islands (Finland)</td>\n",
       "      <td>Mariehamn</td>\n",
       "      <td>Jomala</td>\n",
       "      <td>[citation needed]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Albania</td>\n",
       "      <td>Tirana</td>\n",
       "      <td>Durrës</td>\n",
       "      <td>[2]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Algeria</td>\n",
       "      <td>Algiers</td>\n",
       "      <td>Oran</td>\n",
       "      <td>[3]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>American Samoa (US)</td>\n",
       "      <td>Pago Pago</td>\n",
       "      <td>Tafuna</td>\n",
       "      <td>Nu'uuli</td>\n",
       "      <td>[4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Andorra</td>\n",
       "      <td>Andorra la Vella</td>\n",
       "      <td>Les Escaldes</td>\n",
       "      <td>[5]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Angola</td>\n",
       "      <td>Luanda</td>\n",
       "      <td>Huambo</td>\n",
       "      <td>[6]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Anguilla (UK)</td>\n",
       "      <td>The Valley</td>\n",
       "      <td>North Side</td>\n",
       "      <td>The Valley</td>\n",
       "      <td>[7]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Antigua and Barbuda</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>All Saints</td>\n",
       "      <td>[8]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Argentina</td>\n",
       "      <td>Buenos Aires</td>\n",
       "      <td>Córdoba</td>\n",
       "      <td>[9]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Armenia</td>\n",
       "      <td>Yerevan</td>\n",
       "      <td>Gyumri</td>\n",
       "      <td>[10]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Artsakh (Azerbaijan)</td>\n",
       "      <td>Stepanakert</td>\n",
       "      <td>Shusha</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Aruba (Netherlands)</td>\n",
       "      <td>Oranjestad</td>\n",
       "      <td>Sint Nicolaas</td>\n",
       "      <td>[11]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Canberra</td>\n",
       "      <td>Sydney</td>\n",
       "      <td>Melbourne</td>\n",
       "      <td>[12]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Austria</td>\n",
       "      <td>Vienna</td>\n",
       "      <td>Graz</td>\n",
       "      <td>[13]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>Baku</td>\n",
       "      <td>Ganja</td>\n",
       "      <td>[14]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Bahamas</td>\n",
       "      <td>Nassau</td>\n",
       "      <td>Freeport</td>\n",
       "      <td>[15]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Bahrain</td>\n",
       "      <td>Manama</td>\n",
       "      <td>Riffa</td>\n",
       "      <td>[16]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>Dhaka</td>\n",
       "      <td>Chittagong</td>\n",
       "      <td>[17]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Barbados</td>\n",
       "      <td>Bridgetown</td>\n",
       "      <td>Speightstown</td>\n",
       "      <td>[18]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Belarus</td>\n",
       "      <td>Minsk</td>\n",
       "      <td>Homel</td>\n",
       "      <td>[19]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Belgium</td>\n",
       "      <td>Brussels</td>\n",
       "      <td>Antwerp</td>\n",
       "      <td>[20]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Belize</td>\n",
       "      <td>Belmopan</td>\n",
       "      <td>Belize City</td>\n",
       "      <td>San Ignacio</td>\n",
       "      <td>[21]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Benin</td>\n",
       "      <td>Porto-Novo</td>\n",
       "      <td>Cotonou</td>\n",
       "      <td>Abomey-Calavi</td>\n",
       "      <td>[22]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>Togo</td>\n",
       "      <td>Lomé</td>\n",
       "      <td>Sokodé</td>\n",
       "      <td>[221]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>Tokelau (New Zealand)</td>\n",
       "      <td>None</td>\n",
       "      <td>Fakaofo</td>\n",
       "      <td>Atafu</td>\n",
       "      <td>[222]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Tonga</td>\n",
       "      <td>Nukuʻalofa</td>\n",
       "      <td>Neiafu</td>\n",
       "      <td>[223]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Transnistria (Moldova)</td>\n",
       "      <td>Tiraspol</td>\n",
       "      <td>Tighina</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>Trinidad and Tobago</td>\n",
       "      <td>Port of Spain</td>\n",
       "      <td>Chaguanas</td>\n",
       "      <td>San Fernando</td>\n",
       "      <td>[224]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Tunisia</td>\n",
       "      <td>Tunis</td>\n",
       "      <td>Sfax</td>\n",
       "      <td>[225]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Turkey</td>\n",
       "      <td>Ankara</td>\n",
       "      <td>Istanbul</td>\n",
       "      <td>Ankara</td>\n",
       "      <td>[226]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Turkmenistan</td>\n",
       "      <td>Ashgabat</td>\n",
       "      <td>Türkmenabat</td>\n",
       "      <td>[227]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Turks and Caicos Islands (UK)</td>\n",
       "      <td>Cockburn Town</td>\n",
       "      <td>Five Cays</td>\n",
       "      <td>Honda Road</td>\n",
       "      <td>[228]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Tuvalu</td>\n",
       "      <td>Funafuti[Note 5]</td>\n",
       "      <td>Asau</td>\n",
       "      <td>[229]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Uganda</td>\n",
       "      <td>Kampala</td>\n",
       "      <td>Gulu</td>\n",
       "      <td>[230]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>Kiev</td>\n",
       "      <td>Kharkiv</td>\n",
       "      <td>[231][232]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>Donetsk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>United Arab Emirates</td>\n",
       "      <td>Abu Dhabi</td>\n",
       "      <td>Dubai</td>\n",
       "      <td>Abu Dhabi</td>\n",
       "      <td>[233][234]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>London</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>[235] [236] [237][238] [239]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>United States of America</td>\n",
       "      <td>Washington, D.C.</td>\n",
       "      <td>New York City</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>[240]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>Uruguay</td>\n",
       "      <td>Montevideo</td>\n",
       "      <td>Ciudad de la Costa</td>\n",
       "      <td>[241]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>Maldonado</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>Uzbekistan</td>\n",
       "      <td>Tashkent</td>\n",
       "      <td>Namangan</td>\n",
       "      <td>[242]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Vanuatu</td>\n",
       "      <td>Port Vila</td>\n",
       "      <td>Luganville</td>\n",
       "      <td>[243]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>Vatican City</td>\n",
       "      <td>Vatican City</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[244]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>Venezuela</td>\n",
       "      <td>Caracas</td>\n",
       "      <td>Maracaibo</td>\n",
       "      <td>[245]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Hanoi</td>\n",
       "      <td>Ho Chi Minh City</td>\n",
       "      <td>Hanoi</td>\n",
       "      <td>[246]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>Virgin Islands, British (UK)</td>\n",
       "      <td>Road Town</td>\n",
       "      <td>Spanish Town</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>Virgin Islands, United States (US)</td>\n",
       "      <td>Charlotte Amalie</td>\n",
       "      <td>Anna's Retreat</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>Wallis and Futuna (France)</td>\n",
       "      <td>Mata-Utu</td>\n",
       "      <td>Alele</td>\n",
       "      <td>[247]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>Western Sahara</td>\n",
       "      <td>Laayoune</td>\n",
       "      <td>Dakhla</td>\n",
       "      <td>[248]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>Yemen</td>\n",
       "      <td>Sana'a</td>\n",
       "      <td>Aden</td>\n",
       "      <td>[249]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>Zambia</td>\n",
       "      <td>Lusaka</td>\n",
       "      <td>Kitwe</td>\n",
       "      <td>[250]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>Zimbabwe</td>\n",
       "      <td>Harare</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>[251]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>265 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0                    1  \\\n",
       "0                  Country or territory              Capital   \n",
       "1                               Largest       Second largest   \n",
       "2      Metropolitan area (if different)                  NaN   \n",
       "3                               Largest       Second largest   \n",
       "4                    Abkhazia (Georgia)              Sukhumi   \n",
       "5                           Afghanistan                Kabul   \n",
       "6            Akrotiri and Dhekelia (UK)  Episkopi Cantonment   \n",
       "7               Åland Islands (Finland)            Mariehamn   \n",
       "8                               Albania               Tirana   \n",
       "9                               Algeria              Algiers   \n",
       "10                  American Samoa (US)            Pago Pago   \n",
       "11                              Andorra     Andorra la Vella   \n",
       "12                               Angola               Luanda   \n",
       "13                        Anguilla (UK)           The Valley   \n",
       "14                  Antigua and Barbuda           St. John's   \n",
       "15                            Argentina         Buenos Aires   \n",
       "16                              Armenia              Yerevan   \n",
       "17                 Artsakh (Azerbaijan)          Stepanakert   \n",
       "18                  Aruba (Netherlands)           Oranjestad   \n",
       "19                            Australia             Canberra   \n",
       "20                              Austria               Vienna   \n",
       "21                           Azerbaijan                 Baku   \n",
       "22                              Bahamas               Nassau   \n",
       "23                              Bahrain               Manama   \n",
       "24                           Bangladesh                Dhaka   \n",
       "25                             Barbados           Bridgetown   \n",
       "26                              Belarus                Minsk   \n",
       "27                              Belgium             Brussels   \n",
       "28                               Belize             Belmopan   \n",
       "29                                Benin           Porto-Novo   \n",
       "..                                  ...                  ...   \n",
       "235                                Togo                 Lomé   \n",
       "236               Tokelau (New Zealand)                 None   \n",
       "237                               Tonga           Nukuʻalofa   \n",
       "238              Transnistria (Moldova)             Tiraspol   \n",
       "239                 Trinidad and Tobago        Port of Spain   \n",
       "240                             Tunisia                Tunis   \n",
       "241                              Turkey               Ankara   \n",
       "242                        Turkmenistan             Ashgabat   \n",
       "243       Turks and Caicos Islands (UK)        Cockburn Town   \n",
       "244                              Tuvalu     Funafuti[Note 5]   \n",
       "245                              Uganda              Kampala   \n",
       "246                             Ukraine                 Kiev   \n",
       "247                             Donetsk                  NaN   \n",
       "248                United Arab Emirates            Abu Dhabi   \n",
       "249                      United Kingdom               London   \n",
       "250            United States of America     Washington, D.C.   \n",
       "251                             Uruguay           Montevideo   \n",
       "252                           Maldonado                  NaN   \n",
       "253                          Uzbekistan             Tashkent   \n",
       "254                             Vanuatu            Port Vila   \n",
       "255                        Vatican City         Vatican City   \n",
       "256                           Venezuela              Caracas   \n",
       "257                             Vietnam                Hanoi   \n",
       "258        Virgin Islands, British (UK)            Road Town   \n",
       "259  Virgin Islands, United States (US)     Charlotte Amalie   \n",
       "260          Wallis and Futuna (France)             Mata-Utu   \n",
       "261                      Western Sahara             Laayoune   \n",
       "262                               Yemen               Sana'a   \n",
       "263                              Zambia               Lusaka   \n",
       "264                            Zimbabwe               Harare   \n",
       "\n",
       "                      2                             3           4  \n",
       "0           City proper                        Source         NaN  \n",
       "1                   NaN                           NaN         NaN  \n",
       "2                   NaN                           NaN         NaN  \n",
       "3                   NaN                           NaN         NaN  \n",
       "4                 Gagra                           [1]         NaN  \n",
       "5              Kandahar             [citation needed]         NaN  \n",
       "6        Ayios Nikolaos             [citation needed]         NaN  \n",
       "7                Jomala             [citation needed]         NaN  \n",
       "8                Durrës                           [2]         NaN  \n",
       "9                  Oran                           [3]         NaN  \n",
       "10               Tafuna                       Nu'uuli         [4]  \n",
       "11         Les Escaldes                           [5]         NaN  \n",
       "12               Huambo                           [6]         NaN  \n",
       "13           North Side                    The Valley         [7]  \n",
       "14           All Saints                           [8]         NaN  \n",
       "15              Córdoba                           [9]         NaN  \n",
       "16               Gyumri                          [10]         NaN  \n",
       "17               Shusha                           NaN         NaN  \n",
       "18        Sint Nicolaas                          [11]         NaN  \n",
       "19               Sydney                     Melbourne        [12]  \n",
       "20                 Graz                          [13]         NaN  \n",
       "21                Ganja                          [14]         NaN  \n",
       "22             Freeport                          [15]         NaN  \n",
       "23                Riffa                          [16]         NaN  \n",
       "24           Chittagong                          [17]         NaN  \n",
       "25         Speightstown                          [18]         NaN  \n",
       "26                Homel                          [19]         NaN  \n",
       "27              Antwerp                          [20]         NaN  \n",
       "28          Belize City                   San Ignacio        [21]  \n",
       "29              Cotonou                 Abomey-Calavi        [22]  \n",
       "..                  ...                           ...         ...  \n",
       "235              Sokodé                         [221]         NaN  \n",
       "236             Fakaofo                         Atafu       [222]  \n",
       "237              Neiafu                         [223]         NaN  \n",
       "238             Tighina                           NaN         NaN  \n",
       "239           Chaguanas                  San Fernando       [224]  \n",
       "240                Sfax                         [225]         NaN  \n",
       "241            Istanbul                        Ankara       [226]  \n",
       "242         Türkmenabat                         [227]         NaN  \n",
       "243           Five Cays                    Honda Road       [228]  \n",
       "244                Asau                         [229]         NaN  \n",
       "245                Gulu                         [230]         NaN  \n",
       "246             Kharkiv                    [231][232]         NaN  \n",
       "247                 NaN                           NaN         NaN  \n",
       "248               Dubai                     Abu Dhabi  [233][234]  \n",
       "249          Birmingham  [235] [236] [237][238] [239]         NaN  \n",
       "250       New York City                   Los Angeles       [240]  \n",
       "251  Ciudad de la Costa                         [241]         NaN  \n",
       "252                 NaN                           NaN         NaN  \n",
       "253            Namangan                         [242]         NaN  \n",
       "254          Luganville                         [243]         NaN  \n",
       "255                 NaN                         [244]         NaN  \n",
       "256           Maracaibo                         [245]         NaN  \n",
       "257    Ho Chi Minh City                         Hanoi       [246]  \n",
       "258        Spanish Town                           NaN         NaN  \n",
       "259      Anna's Retreat                           NaN         NaN  \n",
       "260               Alele                         [247]         NaN  \n",
       "261              Dakhla                         [248]         NaN  \n",
       "262                Aden                         [249]         NaN  \n",
       "263               Kitwe                         [250]         NaN  \n",
       "264            Bulawayo                         [251]         NaN  \n",
       "\n",
       "[265 rows x 5 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wiki[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_csv(wiki[1],'/Users/adityakumar/Desktop/wiki.csv',encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 265 entries, 0 to 264\n",
      "Data columns (total 6 columns):\n",
      "Unnamed: 0    265 non-null int64\n",
      "0             265 non-null object\n",
      "1             253 non-null object\n",
      "2             242 non-null object\n",
      "3             230 non-null object\n",
      "4             53 non-null object\n",
      "dtypes: int64(1), object(5)\n",
      "memory usage: 12.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('/Users/adityakumar/Desktop/wiki.csv')\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.to_csv(ds,'/Users/adityakumar/Desktop/wiki.csv',encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds =ds.dropna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}