ADS_Week3_Task1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Assignment Week 3 Task 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transfering Data in Padas dataframe using BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Installing BeautifulSoup Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: BeautifulSoup4 in /srv/conda/envs/notebook/lib/python3.7/site-packages (4.8.1)\n",
      "Requirement already satisfied: soupsieve>=1.2 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from BeautifulSoup4) (1.9.4)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install BeautifulSoup4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### importing required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fetching data from Wikipedia\n",
    "source = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(source, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M2A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Harbourfront\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  PostCode           Borough        Neighborhood\n",
       "0      M1A      Not assigned      Not assigned\\n\n",
       "1      M2A      Not assigned      Not assigned\\n\n",
       "2      M3A        North York         Parkwoods\\n\n",
       "3      M4A        North York  Victoria Village\\n\n",
       "4      M5A  Downtown Toronto      Harbourfront\\n"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting data into pandas dataframe\n",
    "\n",
    "data = soup.find('table', class_='wikitable sortable').tbody\n",
    "# print(data)\n",
    "\n",
    "columns = ['PostCode', 'Borough', 'Neighborhood']\n",
    "toronto_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "for row in data.find_all('tr'):\n",
    "    row_data = []\n",
    "    for col in row.find_all('td'):\n",
    "        row_data.append(col.text)\n",
    "        \n",
    "    if row_data:\n",
    "        df = pd.DataFrame({\"PostCode\" : [row_data[0]],\n",
    "                      \"Borough\" : [row_data[1]],\n",
    "                      \"Neighborhood\" : [row_data[2]]})\n",
    "        toronto_df = toronto_df.append(df, ignore_index=True)\n",
    "toronto_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cleaning the data\n",
    "toronto_df['Neighborhood'] = [value.split('\\n')[0] for value in toronto_df['Neighborhood']]\n",
    "\n",
    "# Removing all rows having 'Not assigned' value to both Neighborhood and Borough column\n",
    "toronto_df = toronto_df[(toronto_df.Neighborhood != 'Not assigned') | (toronto_df.Borough != 'Not assigned')]\n",
    "\n",
    "# Assigning Borough to Neighborhood columns having values as 'Not assigned'\n",
    "toronto_df.loc[toronto_df.Neighborhood == 'Not assigned', 'Neighborhood'] = toronto_df['Borough']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging rows having same 'PostCode' values\n",
    "toronto_df = toronto_df.groupby('PostCode', as_index=False).agg(','.join)\n",
    "toronto_df['Borough'] = toronto_df['Borough'].str.split(',').str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Rouge,Malvern</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Highland Creek,Rouge Hill,Port Union</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Guildwood,Morningside,West Hill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1G</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Woburn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1H</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cedarbrae</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M1J</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M1K</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>East Birchmount Park,Ionview,Kennedy Park</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>M1L</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Clairlea,Golden Mile,Oakridge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>M1M</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cliffcrest,Cliffside,Scarborough Village West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>M1N</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Birch Cliff,Cliffside West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>M1P</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Dorset Park,Scarborough Town Centre,Wexford He...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M1R</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Maryvale,Wexford</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PostCode      Borough                                       Neighborhood\n",
       "0       M1B  Scarborough                                      Rouge,Malvern\n",
       "1       M1C  Scarborough               Highland Creek,Rouge Hill,Port Union\n",
       "2       M1E  Scarborough                    Guildwood,Morningside,West Hill\n",
       "3       M1G  Scarborough                                             Woburn\n",
       "4       M1H  Scarborough                                          Cedarbrae\n",
       "5       M1J  Scarborough                                Scarborough Village\n",
       "6       M1K  Scarborough          East Birchmount Park,Ionview,Kennedy Park\n",
       "7       M1L  Scarborough                      Clairlea,Golden Mile,Oakridge\n",
       "8       M1M  Scarborough      Cliffcrest,Cliffside,Scarborough Village West\n",
       "9       M1N  Scarborough                         Birch Cliff,Cliffside West\n",
       "10      M1P  Scarborough  Dorset Park,Scarborough Town Centre,Wexford He...\n",
       "11      M1R  Scarborough                                   Maryvale,Wexford"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toronto_df.head(12)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103, 3)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toronto_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### This fulfils the assignment requirement of showing that there are 103 rows and 3 columns to this data frame from which the first 12 rows have been displayed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}