database.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import requests, json, re\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_soup(article_data : dict) -> BeautifulSoup:\n",
    "\n",
    "    doc_id          = article_data.get('articleNumber')\n",
    "\n",
    "    url             = 'https://ieeexplore.ieee.org/document/' + doc_id\n",
    "\n",
    "    headers         = {\"User-Agent\":'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0'}\n",
    "\n",
    "    req             = requests.get(url, headers=headers)\n",
    "\n",
    "    soup            = BeautifulSoup(req.text, \"lxml\")\n",
    "\n",
    "    return soup\n",
    "\n",
    "\n",
    "def get_abstract(soup : BeautifulSoup) -> str:\n",
    "\n",
    "    abs_meta    = soup.select_one('meta[property=\"og:description\"]')\n",
    "\n",
    "    abstract    = abs_meta['content'] if abs_meta else \" \"\n",
    "\n",
    "    return abstract\n",
    "\n",
    "\n",
    "def get_keywords(soup : BeautifulSoup, index : int) -> list:\n",
    "\n",
    "    s               = re.findall(r\"xplGlobal\\.document\\.metadata=(.*?)};\", str(soup.select(\"script\")))\n",
    "\n",
    "    if s:\n",
    "\n",
    "        s           = s[0]\n",
    "\n",
    "        meta_dict   = json.loads(s +  \"}\")\n",
    "\n",
    "        try:\n",
    "            ieee_keywords       = meta_dict['keywords'][0]['kwd']  # index 0 = IEEE Keywords, 1 = Index Terms, 2 = Author keywords (not always available)\n",
    "        except:\n",
    "            ieee_keywords       = []\n",
    "\n",
    "\n",
    "        \n",
    "        try:\n",
    "            author_keywords     = meta_dict['keywords'][2]['kwd']# index 0 = IEEE Keywords, 1 = Index Terms, 2 = Author keywords (not always available)\n",
    "        except:\n",
    "            author_keywords     = []\n",
    "\n",
    "        kw_lower    = [keyword.lower() for keyword in ieee_keywords + author_keywords]\n",
    "\n",
    "    else:\n",
    "\n",
    "        #print(soup)\n",
    "        \n",
    "        kw_lower    = []\n",
    "\n",
    "    return kw_lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url             = 'https://ieeexplore.ieee.org/rest/search'\n",
    "\n",
    "headers         = { \"User-Agent\":'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0',\n",
    "                    \"Referer\": \"https://ieeexplore.ieee.org/search/searchresult.jsp?action=search&newsearch=true&matchBoolean=true\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload_mixed  = {\"action\":\"search\",\"newsearch\":True,\"matchBoolean\":True,\"queryText\":\"((\\\"Publication Number\\\":63 \\nOR \\\"Publication Number\\\":8782709 \\nOR \\\"Parent Publication Number\\\":1842944 \\nOR \\\"Parent Publication Number\\\":1000047 \\nOR \\\"Parent Publication Number\\\":1002943 \\nOR \\\"Parent Publication Number\\\":1001483 \\nOR \\\"Parent Publication Number\\\":1000158\\n)\\nAND\\n(\\n \\\"Abstract\\\":\\\"design automation\\\"\\nOR  \\\"Abstract\\\":\\\"automated\\\"\\nOR \\\"Abstract\\\":\\\"optimization algorithm\\\"\\nOR \\\"Abstract\\\":\\\"optimization technique\\\"\\nOR \\\"Abstract\\\":\\\"design optimization\\\"\\nOR \\\"Abstract\\\":\\\"optimizer\\\"\\nOR \\\"Abstract\\\":\\\"design strategy\\\"\\nOR \\\"Abstract\\\":\\\"design procedure\\\"\\nOR \\\"Abstract\\\":\\\"design method\\\"\\nOR \\\"Abstract\\\":\\\"design methodology\\\"\\nOR \\\"Abstract\\\":\\\"design tool\\\"\\nOR \\\"Abstract\\\":\\\"design flow\\\"\\nOR \\\"Abstract\\\":\\\"design space\\\"\\nOR \\\"Abstract\\\":\\\"script\\\"\\nOR \\\"Abstract\\\":\\\"synthesis\\\"\\nOR \\\"Abstract\\\":\\\"algorithm\\\"\\nOR \\\"Abstract\\\":\\\"computer-aided\\\"\\n))\"\n",
    ",\"highlight\":True,\"returnFacets\":[\"ALL\"],\"returnType\":\"SEARCH\",\"matchPubs\":True,\"rowsPerPage\": \"100\"}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_info(article : dict) -> dict:\n",
    "\n",
    "    if article:\n",
    "\n",
    "        soup        = get_soup(article)\n",
    "\n",
    "        #print(soup.title.string)\n",
    "\n",
    "        if soup.title.string == \"Request Rejected\":\n",
    "\n",
    "            print(\"cats\")\n",
    "\n",
    "            #time.sleep(30)\n",
    "\n",
    "            return get_all_info(article)\n",
    "        \n",
    "        else:\n",
    "\n",
    "            keywords    = get_keywords(soup, index = 0)\n",
    "\n",
    "            abstract    = get_abstract(soup)\n",
    "\n",
    "            result      = { 'conference'   : article['publicationTitle']\n",
    "                        ,   'year'         : article['publicationYear']\n",
    "                        ,   'title'        : article['articleTitle']\n",
    "                        ,   'citations'    : article['citationCount']\n",
    "                        ,   'keywords'     : keywords\n",
    "                        ,   'abstract'     : abstract\n",
    "                        }\n",
    "    else:\n",
    "\n",
    "        result      = {}\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "def scrape(payload : dict, num_workers : int) -> pd.DataFrame:\n",
    "\n",
    "    #print(f\"Payload Number: {i}\")\n",
    "\n",
    "    all_pages                       = []\n",
    "\n",
    "    payload['pageNumber']           = 1\n",
    "\n",
    "    while True:\n",
    "\n",
    "        req                         = requests.post(url, headers=headers, json=payload)\n",
    "\n",
    "        #print(req)\n",
    "\n",
    "        query_result                = req.json()['records'] if 'records' in req.json() else [{}]\n",
    "\n",
    "        if len(query_result) == 1:\n",
    "            break\n",
    "        else:\n",
    "            all_pages               = all_pages + query_result\n",
    "            payload['pageNumber']   = payload['pageNumber'] + 1\n",
    "\n",
    "    print(f\"Number of articles: {len(all_pages)}.\")\n",
    "\n",
    "    #print(f\"No. of articles in that payload: {len(all_pages)}\")\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_workers) as executor:\n",
    "        futures = [executor.submit(get_all_info, article) for article in all_pages]\n",
    "\n",
    "\n",
    "    return pd.DataFrame([future.result() for future in futures])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result_df       = pd.DataFrame()\n",
    "\n",
    "# for j, payload in enumerate(payloads):\n",
    "\n",
    "#     print(f\"Payload No. {j}\")\n",
    "\n",
    "#     df_venue    = scrape(payload, num_workers=100)\n",
    "\n",
    "#     filename    = f\"df_{j}.pkl\"\n",
    "\n",
    "#     df_venue.to_pickle(filename)\n",
    "\n",
    "#     #result_df   = pd.concat([result_df, df_venue])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The IEEE Xplore site might be not responding and a JSON Decode Error occurs. Do one payload at a time and give cooldown time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of articles: 6532.\n"
     ]
    }
   ],
   "source": [
    "df_venue    = scrape(payload_mixed, num_workers=3) # Do not use more than 2 workers, otherwise IEEE Xplore becomes unresponsive\n",
    "\n",
    "filename    = \"df_mixed.pkl\"\n",
    "\n",
    "df_venue.to_pickle(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.read_pickle('df_mixed.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# url             = 'https://ieeexplore.ieee.org/document/' + '10509331'\n",
    "\n",
    "# headers         = {\"User-Agent\":'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0'}\n",
    "\n",
    "# req             = requests.get(url, headers=headers)\n",
    "\n",
    "# soup            = BeautifulSoup(req.text, \"lxml\")\n",
    "\n",
    "# s               = re.findall(r\"xplGlobal\\.document\\.metadata=(.*?)};\", str(soup.select(\"script\")))\n",
    "\n",
    "# if s:\n",
    "\n",
    "#     s           = s[0]\n",
    "\n",
    "#     meta_dict   = json.loads(s +  \"}\")\n",
    "\n",
    "#     try:\n",
    "#         ieee_keywords       = meta_dict['keywords'][0]['kwd']  # index 0 = IEEE Keywords, 1 = Index Terms, 2 = Author keywords (not always available)\n",
    "#     except:\n",
    "#         ieee_keywords       = []\n",
    "\n",
    "\n",
    "    \n",
    "#     try:\n",
    "#         author_keywords     = meta_dict['keywords'][2]['kwd']# index 0 = IEEE Keywords, 1 = Index Terms, 2 = Author keywords (not always available)\n",
    "#     except:\n",
    "#         author_keywords     = []\n",
    "\n",
    "# print(author_keywords, ieee_keywords)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ape",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}