Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6e3ff8a6",
   "metadata": {},
   "source": [
    "# Goodreads Web Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dd33c33",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "import urllib.request\n",
    "import Goodreads_helper_functions as good\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c9a4e7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_book_urls(url):\n",
    "\n",
    "    urls = []\n",
    "\n",
    "    for i in range(1,20):\n",
    "        new_url = str(url) + f'?page={i}'\n",
    "        open_url = urllib.request.urlopen(new_url)\n",
    "        soup = bs(open_url, 'html.parser')\n",
    "        soups = soup.find_all('div', {\"data-resource-type\":\"Book\"})\n",
    "\n",
    "        for i in range(len(soups)):\n",
    "            urls.append('https://goodreads.com' + soups[i].a['href'])\n",
    "\n",
    "    return urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e93b56e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "url17='https://www.goodreads.com/list/show/107026.Best_Books_of_2017'\n",
    "url18='https://www.goodreads.com/list/show/119307.Best_Books_of_2018'\n",
    "urls2018 = get_book_urls(url18)\n",
    "urls2017 = get_book_urls(url17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39b5e3c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_book_info(urls):\n",
    "\n",
    "    books = []\n",
    "    # iterate over the list of urls\n",
    "    for url in urls:\n",
    "        page = requests.get(url)\n",
    "        soup = bs(page.content, 'html.parser')\n",
    "    \n",
    "        book_dict = {}\n",
    "        book_dict['title'] = good.get_title(soup)\n",
    "        book_dict['ISBN'] = good.get_ISBN(soup)\n",
    "        book_dict['author'] = good.get_author(soup)\n",
    "        book_dict['series'] = good.get_series(soup)\n",
    "        book_dict['genre'] = good.get_genre(soup)\n",
    "        book_dict['rating'] = good.get_rating(soup)\n",
    "        book_dict['publish_date'] = good.get_publish_date(soup)\n",
    "        book_dict['publish_company'] = good.get_publishing_company(soup)\n",
    "        book_dict['number_of_pages'] = good.get_pages(soup)\n",
    "        book_dict['format'] = good.get_format(soup)\n",
    "    \n",
    "        books.append(book_dict)\n",
    "    return books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e64ab58",
   "metadata": {},
   "outputs": [],
   "source": [
    "book_dict_2018 = get_book_info(urls2018)\n",
    "book_dict_2017 = get_book_info(urls2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c66847b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Goodreads_books_2017.json','w') as book_file:\n",
    "    json.dump(book_dict_2017, book_file)\n",
    "with open('Goodreads_books_2018.json','w') as book_file:\n",
    "    json.dump(book_dict_2018, book_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dc2c0ef",
   "metadata": {},
   "source": [
    "# NYT Web Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8b8c0e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import config\n",
    "import json\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import Goodreads_helper_functions as good\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eebf730c",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_sellers = []\n",
    "\n",
    "def get_books():\n",
    "    \"\"\"\n",
    "    get_books():\n",
    "    Function returns a list of New York Times bestsellers\n",
    "    Params:\n",
    "        None\n",
    "    Returns:\n",
    "        List of dictionaries describing bestselling books\n",
    "    \"\"\"\n",
    "    # offset the pages by multiples of 20\n",
    "    for i in range(0, 32326, 20):\n",
    "        params = {'api-key': config.NYT_api_key,\n",
    "                 'offset': i}\n",
    "        url = 'https://api.nytimes.com/svc/books/v3/lists/best-sellers/history.json'\n",
    "        response = requests.get(url, params = params)\n",
    "        data = response.json()\n",
    "        \n",
    "        best_sellers = {}\n",
    "       \n",
    "        for book in data['results']:\n",
    "            try:\n",
    "\n",
    "                best_sellers['title'] = book['title']\n",
    "                best_sellers['author'] = book['author']\n",
    "                best_sellers['publisher'] = book['publisher']\n",
    "                best_sellers['ISBN'] = book['isbns']\n",
    "                best_sellers['publish_date'] = book['ranks_history'][0]['published_date']\n",
    "\n",
    "                # add the dictionary to master list\n",
    "\n",
    "                best_sellers.append(best_sellers_dict)\n",
    "\n",
    "            except IndexError:\n",
    "                continue\n",
    "\n",
    "        # print what page we are on for auditing purposes\n",
    "        print(i)\n",
    "        \n",
    "        # wait so we don't hit the API's per minute call limit\n",
    "        time.sleep(6)\n",
    "            \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b495788",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59d8a385",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_NYT_ISBNS(data):\n",
    "    \"\"\"\n",
    "    get_NYT_ISBNS():\n",
    "    Function returns a list of ISBNs from the list of NYT bestsellers between 2017 and the present\n",
    "    Params:\n",
    "        data: list of dictionaries describing each book from the NYT API call\n",
    "    Returns:\n",
    "        List of ISBNs for each NYT bestseller\n",
    "    \"\"\"\n",
    "    ISBNs = []\n",
    "    years = ['2019','2018','2017']\n",
    "    for book in data:\n",
    "        # only append the ISBN if the book was on a NYT bestseller list between 2017 and 2019\n",
    "        if any(x in book['publish_date'] for x in years):\n",
    "            try: \n",
    "                ISBN = book['ISBN'][0]['isbn13']\n",
    "                ISBNs.append(ISBN)\n",
    "            except IndexError:\n",
    "                continue\n",
    "    return ISBNs\n",
    "\n",
    "ISBNs = get_NYT_ISBNS(best_sellers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1e62cf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver_options = webdriver.chrome.options.Options()\n",
    "img = {\"profile.managed_default_content_settings.images\": 2}\n",
    "driver_options.add_experimental_option(\"prefs\", img)\n",
    "driver = webdriver.Chrome(options=driver_options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70d8dc02",
   "metadata": {},
   "outputs": [],
   "source": [
    "NYT_books = []\n",
    "\n",
    "def get_NYTbook_info(ISBNs):\n",
    "    \"\"\"\n",
    "    get_NYTbook_info():\n",
    "    Function returns a list of dictionaries with features describing each NYT bestseller\n",
    "    Params:\n",
    "        ISBNs: list of ISBNs for each NYT bestseller\n",
    "    Returns:\n",
    "        List of dictionaries describing bestselling books\n",
    "    \"\"\"\n",
    "\n",
    "    for ISBN in ISBNs:\n",
    "        \n",
    "        # log into goodreads website \n",
    "        driver.get('https://www.goodreads.com/')\n",
    "        \n",
    "\n",
    "        try:\n",
    "            # get the webpage for each ISBN \n",
    "            driver.find_element_by_xpath('/html/body/div[4]/main/div[1]/section[1]/div/div/footer/div[1]/div/form/input').send_keys(ISBN,Keys.ENTER)\n",
    "            time.sleep(2) \n",
    "\n",
    "            # grab the current url to scrape\n",
    "            url = driver.current_url\n",
    "            \n",
    "        # Beautiful soup to scrape each book page for features\n",
    "            \n",
    "            html = requests.get(url)\n",
    "            soup = bs(html.content, 'html.parser')\n",
    "\n",
    "            book = {}\n",
    "\n",
    "            book['title'] = good.get_title(soup)\n",
    "            book['ISBN'] = ISBN\n",
    "            book['author'] = good.get_author(soup)\n",
    "            book['series'] = good.get_series(soup)\n",
    "            book['genre'] = good.get_genre(soup)\n",
    "            book['rating'] = good.get_rating(soup)\n",
    "            book['publish_date'] = good.get_publish_date(soup)\n",
    "            book['publish_company'] = good.get_publishing_company(soup)\n",
    "            book['number_of_pages'] = good.get_pages(soup)\n",
    "            book['format'] = good.get_format(soup)\n",
    "\n",
    "            NYT_books.append(book)\n",
    "\n",
    "        except AttributeError:\n",
    "            continue\n",
    "        \n",
    "        time.sleep(2)\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de69ad82",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_NYTbook_info(ISBNs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c3a42ac",
   "metadata": {},
   "source": [
    "# Helper function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea263a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_title(soup):\n",
    "    title = soup.find('h1',{'id':'bookTitle'}).get_text()\n",
    "    clean_title = title.replace(\"\\n\",\" \").replace(\"  \", \"\")\n",
    "    return clean_title\n",
    "\n",
    "def get_ISBN(soup):\n",
    "    try:\n",
    "        ISBN = soup.find_all('div',class_='infoBoxRowItem')[1].get_text()\n",
    "        clean_ISBN = ISBN.replace(\"\\n\",\" \").replace(\"\\n\",\" \").replace(\" \", \"\").split(\"(\")[1].replace(\")\",\"\").replace(\"ISBN13:\",\"\")\n",
    "    except IndexError:\n",
    "        clean_ISBN = \"\"\n",
    "    return clean_ISBN\n",
    "\n",
    "def get_author(soup):\n",
    "    author = soup.find('span', itemprop='name').get_text()\n",
    "    return author\n",
    "\n",
    "def get_series(soup):\n",
    "    series = soup.find('a',class_='greyText').get_text()\n",
    "    clean_series = series.replace(\"\\n\",\" \").replace(\"  \", \"\").replace(\"(\",\"\").replace(\")\",\"\")\n",
    "    return clean_series\n",
    "\n",
    "def get_genre(soup):\n",
    "    try:\n",
    "        genre = soup.find('a', class_='actionLinkLite bookPageGenreLink')\n",
    "        clean_genre = genre['href'].replace(\"/genres/\",\"\")\n",
    "    except TypeError:\n",
    "        clean_genre = \"\"\n",
    "    return clean_genre\n",
    "\n",
    "def get_rating(soup):\n",
    "    rating = soup.find('span', itemprop='ratingValue').get_text()\n",
    "    clean_rating = rating.replace(\"\\n\",\" \").replace(\" \", \"\")\n",
    "    return clean_rating\n",
    "\n",
    "def get_publish_date(soup):\n",
    "    try:\n",
    "        date = soup.find_all('div',class_='row')[1].get_text()\n",
    "        clean_date = date.replace(\"\\n\",\" \").replace(\"  \",\"\").split('by')[0].replace(' Published ',\"\")\n",
    "    except IndexError:\n",
    "        clean_date = \"\"\n",
    "    return clean_date\n",
    "\n",
    "def get_publishing_company(soup):\n",
    "    try:\n",
    "        company = soup.find_all('div',class_='row')[1].get_text()\n",
    "        clean_company = company.replace(\"\\n\",\" \").replace(\"  \",\"\").split('by ')[1]\n",
    "    except IndexError:\n",
    "        clean_company = \"\"\n",
    "    return clean_company\n",
    "\n",
    "def get_pages(soup):\n",
    "    try:\n",
    "        pages = soup.find('span', itemprop='numberOfPages').get_text().split()[0]\n",
    "    except AttributeError:\n",
    "        pages = \"\"\n",
    "    return pages\n",
    "\n",
    "def get_format(soup):\n",
    "    try: \n",
    "        book_format = soup.find('span',itemprop = 'bookFormat').get_text()\n",
    "    except AttributeError:\n",
    "        book_format = \"\"\n",
    "    return book_format"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:DL]",
   "language": "python",
   "name": "conda-env-DL-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}