Segmenting and Clustering Neighborhoods with Coordinates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting geocoder\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl (98kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 18.7MB/s ta 0:00:01\n",
      "\u001b[?25hCollecting ratelim (from geocoder)\n",
      "  Downloading https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl\n",
      "Requirement already satisfied: requests in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from geocoder) (2.22.0)\n",
      "Collecting future (from geocoder)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n",
      "\u001b[K     |████████████████████████████████| 829kB 30.3MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from geocoder) (1.13.0)\n",
      "Collecting click (from geocoder)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fa/37/45185cb5abbc30d7257104c434fe0b07e5a195a6847506c074527aa599ec/Click-7.0-py2.py3-none-any.whl (81kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 18.0MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: decorator in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from ratelim->geocoder) (4.4.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from requests->geocoder) (1.25.7)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from requests->geocoder) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from requests->geocoder) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from requests->geocoder) (2019.9.11)\n",
      "Building wheels for collected packages: future\n",
      "  Building wheel for future (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/jupyterlab/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n",
      "Successfully built future\n",
      "Installing collected packages: ratelim, future, click, geocoder\n",
      "Successfully installed click-7.0 future-0.18.2 geocoder-1.38.1 ratelim-0.1.6\n",
      "Packages installed.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install geocoder\n",
    "\n",
    "print('Packages installed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: BeautifulSoup4 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (4.8.1)\n",
      "Requirement already satisfied: soupsieve>=1.2 in /home/jupyterlab/conda/envs/python/lib/python3.6/site-packages (from BeautifulSoup4) (1.9.5)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install BeautifulSoup4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # library to handle data in a vectorized manner\n",
    "import pandas as pd # library for data analsysis\n",
    "import geocoder # import geocoder\n",
    "import requests \n",
    "from bs4 import BeautifulSoup \n",
    "\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page Scrapped.\n"
     ]
    }
   ],
   "source": [
    "URL = \"https://www.wikizeroo.org/index.php?q=aHR0cHM6Ly9lbi53aWtpcGVkaWEub3JnL3dpa2kvTGlzdF9vZl9wb3N0YWxfY29kZXNfb2ZfQ2FuYWRhOl9N\"\n",
    "r = requests.get(URL) \n",
    "  \n",
    "soup = BeautifulSoup(r.content, 'html5lib') \n",
    "table = soup.find('div', attrs = {'id':'container'}) \n",
    "\n",
    "# print(soup.prettify()) \n",
    "print('Page Scrapped.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Collected.\n"
     ]
    }
   ],
   "source": [
    "postalCodes = [];\n",
    "boroughs= [];\n",
    "neighborhoods = [];\n",
    "columnNum = 1;\n",
    "passVal = False\n",
    "\n",
    "for row in soup.find_all('td'):\n",
    "    for cell in row:\n",
    "        if cell.string and cell.string[0].isalpha() and len(cell.string) > 2:\n",
    "            passVal = False\n",
    "            if columnNum == 1:\n",
    "                if passVal == False and cell.string[1].isdigit():\n",
    "                    postalCodes.append(cell.string);   \n",
    "                    columnNum = 2\n",
    "                else:\n",
    "                    continue\n",
    "            elif columnNum == 2 :\n",
    "                if cell.string == 'Not assigned':\n",
    "                    passVal = True\n",
    "                    del postalCodes[-1]\n",
    "                    columnNum = 1\n",
    "                    continue\n",
    "                else:\n",
    "                    boroughs.append(cell.string);      \n",
    "                    columnNum = 3\n",
    "            elif columnNum == 3 :\n",
    "                if cell.string == 'Not assigned\\n':\n",
    "                    neighborhoods.append(boroughs[-1])\n",
    "                else:\n",
    "                    neighborhoods.append(cell.string); \n",
    "                columnNum = 1\n",
    "                \n",
    "print('Data Collected.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [PostalCode, Borough, Neighborhood, Latitude, Longitude]\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define the dataframe columns\n",
    "column_names = ['PostalCode', 'Borough', 'Neighborhood', 'Latitude', 'Longitude'] \n",
    "\n",
    "# instantiate the dataframe\n",
    "neighbors = pd.DataFrame(columns=column_names)\n",
    "\n",
    "neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PostalCode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "      <td>43.752420</td>\n",
       "      <td>-79.329242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "      <td>43.730600</td>\n",
       "      <td>-79.313265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Harbourfront</td>\n",
       "      <td>43.650295</td>\n",
       "      <td>-79.359166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Heights</td>\n",
       "      <td>43.723270</td>\n",
       "      <td>-79.451286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Manor</td>\n",
       "      <td>43.723270</td>\n",
       "      <td>-79.451286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>M8Y</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Sunnylea</td>\n",
       "      <td>43.632835</td>\n",
       "      <td>-79.489550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>M8Z</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Kingsway Park South West\\n</td>\n",
       "      <td>43.624630</td>\n",
       "      <td>-79.526950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>M8Z</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Mimico NW</td>\n",
       "      <td>43.624630</td>\n",
       "      <td>-79.526950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>M8Z</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>The Queensway West</td>\n",
       "      <td>43.624630</td>\n",
       "      <td>-79.526950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>M8Z</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Royal York South West\\n</td>\n",
       "      <td>43.624630</td>\n",
       "      <td>-79.526950</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>209 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    PostalCode           Borough                Neighborhood   Latitude  \\\n",
       "0          M3A        North York                   Parkwoods  43.752420   \n",
       "1          M4A        North York            Victoria Village  43.730600   \n",
       "2          M5A  Downtown Toronto                Harbourfront  43.650295   \n",
       "3          M6A        North York            Lawrence Heights  43.723270   \n",
       "4          M6A        North York              Lawrence Manor  43.723270   \n",
       "..         ...               ...                         ...        ...   \n",
       "204        M8Y         Etobicoke                    Sunnylea  43.632835   \n",
       "205        M8Z         Etobicoke  Kingsway Park South West\\n  43.624630   \n",
       "206        M8Z         Etobicoke                   Mimico NW  43.624630   \n",
       "207        M8Z         Etobicoke          The Queensway West  43.624630   \n",
       "208        M8Z         Etobicoke     Royal York South West\\n  43.624630   \n",
       "\n",
       "     Longitude  \n",
       "0   -79.329242  \n",
       "1   -79.313265  \n",
       "2   -79.359166  \n",
       "3   -79.451286  \n",
       "4   -79.451286  \n",
       "..         ...  \n",
       "204 -79.489550  \n",
       "205 -79.526950  \n",
       "206 -79.526950  \n",
       "207 -79.526950  \n",
       "208 -79.526950  \n",
       "\n",
       "[209 rows x 5 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize your variable to None\n",
    "lat_lng_coords = None\n",
    "\n",
    "for data in range(0, len(postalCodes)-1):\n",
    "    code = postalCodes[data]\n",
    "    borough = boroughs[data]\n",
    "    neighborhood_name = neighborhoods[data]\n",
    "    \n",
    "    g = geocoder.arcgis('{}, Toronto, Ontario'.format(code))\n",
    "    lat_lng_coords = g.latlng\n",
    "\n",
    "    neighbors = neighbors.append({ 'PostalCode': code,\n",
    "                                   'Borough': borough,\n",
    "                                   'Neighborhood': neighborhood_name,\n",
    "                                   'Latitude': lat_lng_coords[0],\n",
    "                                   'Longitude': lat_lng_coords[1]}, ignore_index=True)\n",
    "    \n",
    "neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(209, 5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neighbors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python",
   "language": "python",
   "name": "conda-env-python-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}