00_Getting_Data_Urls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5476d50d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "759bfeff",
   "metadata": {},
   "outputs": [],
   "source": [
    "#credit for this code: https://www.hackerearth.com/fr/practice/notes/praveen97uma/crawling-a-website-that-loads-content-using-javascript-with-selenium-webdriver-in-python/\n",
    "#this code creates a function that gets the browser to scroll down\n",
    "def scrollDown(driver, n_scroll):\n",
    "    body = driver.find_element_by_tag_name(\"body\")\n",
    "    while n_scroll >= 0:\n",
    "        body.send_keys(Keys.PAGE_DOWN)\n",
    "        n_scroll -= 1\n",
    "    return driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f14a8c36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 0\n",
      "Progress: 100\n",
      "Progress: 200\n",
      "Progress: 300\n",
      "Progress: 400\n",
      "Progress: 500\n",
      "Progress: 600\n",
      "Progress: 700\n",
      "Progress: 800\n",
      "Progress: 900\n",
      "Progress: 1000\n",
      "Progress: 1100\n",
      "Progress: 1200\n",
      "Progress: 1300\n",
      "Progress: 1400\n",
      "Progress: 1438\n"
     ]
    }
   ],
   "source": [
    "chrome_path = \"C:/Users/Nadir/Desktop/B/Projects/Fragrance_project/chromedriver\"\n",
    "driver = webdriver.Chrome(executable_path = chrome_path)\n",
    "\n",
    "\n",
    "#this list represents different skin care categories to collect\n",
    "categories = ['fragrance']\n",
    "#creating an empty data frame that all the URLs and categories will go in to\n",
    "final_df = pd.DataFrame(columns = ['category', 'URL'])\n",
    "\n",
    "\n",
    "for category in categories: #looping through the different categories\n",
    "\n",
    "    page_num = 1 #the first page\n",
    "\n",
    "    while True:\n",
    "        \n",
    "        print(\"Progress: {}\".format(\"\" + str(len(final_df))) )\n",
    "    \n",
    "        #when running the following .get request in order for everything to work properly\n",
    "        #the window needs to be popped open on the screen\n",
    "        url = 'https://www.sephora.com/shop/'+ category + '?pageSize=300&currentPage=' + str(page_num)\n",
    "        driver.get(url)\n",
    "        time.sleep(15)\n",
    "        try: #check to see if the page is empty   \n",
    "            if driver.find_element_by_class_name('css-2td14d.e65zztl0').is_displayed():\n",
    "                break\n",
    "            \n",
    "        except:\n",
    "    \n",
    "            #check to see if there is a pop up windew\n",
    "            try:\n",
    "                #exit the pop up window\n",
    "                #btn = driver.find_element_by_xpath('//*[@id=\"modalDialog\"]/button').click()\n",
    "                btn = driver.find_element_by_class_name('css-1kna575').click()\n",
    "                time.sleep(10)\n",
    "            except:\n",
    "                pass\n",
    "    \n",
    "            #as scrolling check if there is any more room to scroll\n",
    "            old_len = 0\n",
    "            while True:\n",
    "                browser = scrollDown(driver, 20) #scroll down the page\n",
    "                time.sleep(10) #give it time to load\n",
    "                slug = driver.find_elements_by_class_name('css-klx76') #look for the urls of products\n",
    "                new_len = len(slug)\n",
    "                if old_len == new_len: #if the old length and new length are equal, the bottom of page was reached\n",
    "                    break\n",
    "                else:\n",
    "                    old_len = new_len\n",
    "\n",
    "            #from the list of URLs in slug pull all the href and make a dictionary with it and the category name        \n",
    "            slugURL = []\n",
    "            for a in slug:\n",
    "                subURL = {}\n",
    "                subURL['category'] = category \n",
    "                subURL['URL'] = a.get_attribute('href')\n",
    "                slugURL.append(subURL)\n",
    "    \n",
    "            #append our data frame with categories and URLs\n",
    "            df = pd.DataFrame(slugURL)\n",
    "            #print(df.head())\n",
    "            #adding to go to next page\n",
    "            page_num += 1\n",
    "            #concatenating to get all in same df\n",
    "            final_df = pd.concat([final_df, df], axis = 0, ignore_index = True) \n",
    "    \n",
    "#close the chrome\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6fc91595",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/sol-de-janeiro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/brazilian-crus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/chance-eau-ten...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/yves-saint-lau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/sol-de-janeiro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1433</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/velvet-pear-vo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1434</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/mandarino-di-a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1435</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/ralph-lauren-r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1436</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/19-69-purple-h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1437</th>\n",
       "      <td>fragrance</td>\n",
       "      <td>https://www.sephora.com/product/19-69-capri-ea...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1438 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       category                                                URL\n",
       "0     fragrance  https://www.sephora.com/product/sol-de-janeiro...\n",
       "1     fragrance  https://www.sephora.com/product/brazilian-crus...\n",
       "2     fragrance  https://www.sephora.com/product/chance-eau-ten...\n",
       "3     fragrance  https://www.sephora.com/product/yves-saint-lau...\n",
       "4     fragrance  https://www.sephora.com/product/sol-de-janeiro...\n",
       "...         ...                                                ...\n",
       "1433  fragrance  https://www.sephora.com/product/velvet-pear-vo...\n",
       "1434  fragrance  https://www.sephora.com/product/mandarino-di-a...\n",
       "1435  fragrance  https://www.sephora.com/product/ralph-lauren-r...\n",
       "1436  fragrance  https://www.sephora.com/product/19-69-purple-h...\n",
       "1437  fragrance  https://www.sephora.com/product/19-69-capri-ea...\n",
       "\n",
       "[1438 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking the structure of my final dataframe\n",
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b041861c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the final dataframe of URLs in to a csv folder\n",
    "final_df.to_csv('C:/Users/Nadir/Desktop/B/Projects/Fragrance_project/data/product_urls.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}