Step1-GenerateCorpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: Generate corpus from all (JSON & PMC) files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlpuser/anaconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3146: DtypeWarning: Columns (1,4,5,6,13,14,15,16) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    }
   ],
   "source": [
    "metadata = pd.read_csv('../CORD19/metadata.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select only those records having pdf_json or pmc_json or atleast abstract\n",
    "metadata = metadata[(metadata['pdf_json_files'].notna() | metadata['pmc_json_files'].notna())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cord_uid</th>\n",
       "      <th>sha</th>\n",
       "      <th>source_x</th>\n",
       "      <th>title</th>\n",
       "      <th>doi</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>pubmed_id</th>\n",
       "      <th>license</th>\n",
       "      <th>abstract</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>authors</th>\n",
       "      <th>journal</th>\n",
       "      <th>mag_id</th>\n",
       "      <th>who_covidence_id</th>\n",
       "      <th>arxiv_id</th>\n",
       "      <th>pdf_json_files</th>\n",
       "      <th>pmc_json_files</th>\n",
       "      <th>url</th>\n",
       "      <th>s2_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>764542</th>\n",
       "      <td>4p5r3410</td>\n",
       "      <td>b5881e8c6badd610b7a8f8b59ac207dbd76bb47b</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>Development of Nanobodies Targeting Peste des ...</td>\n",
       "      <td>10.3390/ani11082206</td>\n",
       "      <td>PMC8388416</td>\n",
       "      <td>3.44387e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>SIMPLE SUMMARY: Peste des petits ruminants vir...</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Kinimi, Edson; Muyldermans, Serge; Vincke, Céc...</td>\n",
       "      <td>Animals (Basel)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/b5881e8c6badd610b7a8f...</td>\n",
       "      <td>document_parses/pmc_json/PMC8388416.xml.json</td>\n",
       "      <td>https://doi.org/10.3390/ani11082206; https://w...</td>\n",
       "      <td>237321137.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764543</th>\n",
       "      <td>04pp0o74</td>\n",
       "      <td>28fa600d63823490645957539ed1e2091373388f</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>A Novel Reverse Transcription Loop-Mediated Is...</td>\n",
       "      <td>10.3390/ijms21082826</td>\n",
       "      <td>PMC7216271</td>\n",
       "      <td>3.23256e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>COVID-19 has become a major global public heal...</td>\n",
       "      <td>2020-04-18</td>\n",
       "      <td>Lu, Renfei; Wu, Xiuming; Wan, Zhenzhou; Li, Yi...</td>\n",
       "      <td>Int J Mol Sci</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/28fa600d6382349064595...</td>\n",
       "      <td>document_parses/pmc_json/PMC7216271.xml.json</td>\n",
       "      <td>https://doi.org/10.3390/ijms21082826; https://...</td>\n",
       "      <td>216109792.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764544</th>\n",
       "      <td>g2jx3awo</td>\n",
       "      <td>69363dafd5448d2448f08d0641ad13f476d82e46</td>\n",
       "      <td>Elsevier; Medline; PMC</td>\n",
       "      <td>Suicidal Ideation During the COVID-19 Pandemic...</td>\n",
       "      <td>10.1016/j.psychres.2020.113134</td>\n",
       "      <td>PMC7255187</td>\n",
       "      <td>3.2505e+07</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>There is growing concern over the potential fo...</td>\n",
       "      <td>2020-05-27</td>\n",
       "      <td>Killgore, William D.S.; Cloonen, Sara A.; Tayl...</td>\n",
       "      <td>Psychiatry Res</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/69363dafd5448d2448f08...</td>\n",
       "      <td>document_parses/pmc_json/PMC7255187.xml.json</td>\n",
       "      <td>https://doi.org/10.1016/j.psychres.2020.113134...</td>\n",
       "      <td>218972457.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764545</th>\n",
       "      <td>pgj09tww</td>\n",
       "      <td>96536cef98966bb717a21174d4d4848e9e10f749</td>\n",
       "      <td>Medline; PMC; WHO</td>\n",
       "      <td>Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...</td>\n",
       "      <td>10.1007/s11831-021-09638-y</td>\n",
       "      <td>PMC8381867</td>\n",
       "      <td>3.44566e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>The timing and sequence of safe campus reopeni...</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>Linka, Kevin; Peirlinck, Mathias; Schäfer, Ame...</td>\n",
       "      <td>Arch Comput Methods Eng</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/96536cef98966bb717a21...</td>\n",
       "      <td>document_parses/pmc_json/PMC8381867.xml.json</td>\n",
       "      <td>https://doi.org/10.1007/s11831-021-09638-y; ht...</td>\n",
       "      <td>237279499.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764546</th>\n",
       "      <td>gbs1yj3a</td>\n",
       "      <td>cecc02ad381948150ea4a96a2960d86b473d6882</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>COVID-19 Lockdowns: Impact on Facility-Based H...</td>\n",
       "      <td>10.1007/s10461-020-02939-6</td>\n",
       "      <td>PMC7265663</td>\n",
       "      <td>3.24886e+07</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-02</td>\n",
       "      <td>Mhango, Malizgani; Chitungo, Itai; Dzinamarira...</td>\n",
       "      <td>AIDS Behav</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/cecc02ad381948150ea4a...</td>\n",
       "      <td>document_parses/pmc_json/PMC7265663.xml.json</td>\n",
       "      <td>https://doi.org/10.1007/s10461-020-02939-6; ht...</td>\n",
       "      <td>219176080.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        cord_uid                                       sha  \\\n",
       "764542  4p5r3410  b5881e8c6badd610b7a8f8b59ac207dbd76bb47b   \n",
       "764543  04pp0o74  28fa600d63823490645957539ed1e2091373388f   \n",
       "764544  g2jx3awo  69363dafd5448d2448f08d0641ad13f476d82e46   \n",
       "764545  pgj09tww  96536cef98966bb717a21174d4d4848e9e10f749   \n",
       "764546  gbs1yj3a  cecc02ad381948150ea4a96a2960d86b473d6882   \n",
       "\n",
       "                      source_x  \\\n",
       "764542            Medline; PMC   \n",
       "764543            Medline; PMC   \n",
       "764544  Elsevier; Medline; PMC   \n",
       "764545       Medline; PMC; WHO   \n",
       "764546            Medline; PMC   \n",
       "\n",
       "                                                    title  \\\n",
       "764542  Development of Nanobodies Targeting Peste des ...   \n",
       "764543  A Novel Reverse Transcription Loop-Mediated Is...   \n",
       "764544  Suicidal Ideation During the COVID-19 Pandemic...   \n",
       "764545  Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...   \n",
       "764546  COVID-19 Lockdowns: Impact on Facility-Based H...   \n",
       "\n",
       "                                   doi       pmcid    pubmed_id license  \\\n",
       "764542             10.3390/ani11082206  PMC8388416  3.44387e+07   cc-by   \n",
       "764543            10.3390/ijms21082826  PMC7216271  3.23256e+07   cc-by   \n",
       "764544  10.1016/j.psychres.2020.113134  PMC7255187   3.2505e+07   no-cc   \n",
       "764545      10.1007/s11831-021-09638-y  PMC8381867  3.44566e+07   cc-by   \n",
       "764546      10.1007/s10461-020-02939-6  PMC7265663  3.24886e+07   no-cc   \n",
       "\n",
       "                                                 abstract publish_time  \\\n",
       "764542  SIMPLE SUMMARY: Peste des petits ruminants vir...   2021-07-26   \n",
       "764543  COVID-19 has become a major global public heal...   2020-04-18   \n",
       "764544  There is growing concern over the potential fo...   2020-05-27   \n",
       "764545  The timing and sequence of safe campus reopeni...   2021-08-23   \n",
       "764546                                                NaN   2020-06-02   \n",
       "\n",
       "                                                  authors  \\\n",
       "764542  Kinimi, Edson; Muyldermans, Serge; Vincke, Céc...   \n",
       "764543  Lu, Renfei; Wu, Xiuming; Wan, Zhenzhou; Li, Yi...   \n",
       "764544  Killgore, William D.S.; Cloonen, Sara A.; Tayl...   \n",
       "764545  Linka, Kevin; Peirlinck, Mathias; Schäfer, Ame...   \n",
       "764546  Mhango, Malizgani; Chitungo, Itai; Dzinamarira...   \n",
       "\n",
       "                        journal  mag_id who_covidence_id arxiv_id  \\\n",
       "764542          Animals (Basel)     NaN              NaN      NaN   \n",
       "764543            Int J Mol Sci     NaN              NaN      NaN   \n",
       "764544           Psychiatry Res     NaN              NaN      NaN   \n",
       "764545  Arch Comput Methods Eng     NaN              NaN      NaN   \n",
       "764546               AIDS Behav     NaN              NaN      NaN   \n",
       "\n",
       "                                           pdf_json_files  \\\n",
       "764542  document_parses/pdf_json/b5881e8c6badd610b7a8f...   \n",
       "764543  document_parses/pdf_json/28fa600d6382349064595...   \n",
       "764544  document_parses/pdf_json/69363dafd5448d2448f08...   \n",
       "764545  document_parses/pdf_json/96536cef98966bb717a21...   \n",
       "764546  document_parses/pdf_json/cecc02ad381948150ea4a...   \n",
       "\n",
       "                                      pmc_json_files  \\\n",
       "764542  document_parses/pmc_json/PMC8388416.xml.json   \n",
       "764543  document_parses/pmc_json/PMC7216271.xml.json   \n",
       "764544  document_parses/pmc_json/PMC7255187.xml.json   \n",
       "764545  document_parses/pmc_json/PMC8381867.xml.json   \n",
       "764546  document_parses/pmc_json/PMC7265663.xml.json   \n",
       "\n",
       "                                                      url        s2_id  \n",
       "764542  https://doi.org/10.3390/ani11082206; https://w...  237321137.0  \n",
       "764543  https://doi.org/10.3390/ijms21082826; https://...  216109792.0  \n",
       "764544  https://doi.org/10.1016/j.psychres.2020.113134...  218972457.0  \n",
       "764545  https://doi.org/10.1007/s11831-021-09638-y; ht...  237279499.0  \n",
       "764546  https://doi.org/10.1007/s10461-020-02939-6; ht...  219176080.0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata['date'] = pd.to_datetime(metadata['publish_time'].str.strip(), format='%Y-%m-%d') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = metadata[metadata['date'].dt.year == 2021]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cord_uid</th>\n",
       "      <th>sha</th>\n",
       "      <th>source_x</th>\n",
       "      <th>title</th>\n",
       "      <th>doi</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>pubmed_id</th>\n",
       "      <th>license</th>\n",
       "      <th>abstract</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>authors</th>\n",
       "      <th>journal</th>\n",
       "      <th>mag_id</th>\n",
       "      <th>who_covidence_id</th>\n",
       "      <th>arxiv_id</th>\n",
       "      <th>pdf_json_files</th>\n",
       "      <th>pmc_json_files</th>\n",
       "      <th>url</th>\n",
       "      <th>s2_id</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14597</th>\n",
       "      <td>aoiambb8</td>\n",
       "      <td>3798d3bcb09df782968320b8ad39f28a1bed48e6</td>\n",
       "      <td>PMC</td>\n",
       "      <td>Etiology of Severe Acute Respiratory Infection...</td>\n",
       "      <td>10.3201/eid2701.201347</td>\n",
       "      <td>PMC7774577</td>\n",
       "      <td>33350930</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>In April 2017, surveillance detected a surge i...</td>\n",
       "      <td>2021-01-27</td>\n",
       "      <td>Rahaman, Md R.; Alroy, Karen A.; Van Beneden, ...</td>\n",
       "      <td>Emerg Infect Dis</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/3798d3bcb09df78296832...</td>\n",
       "      <td>document_parses/pmc_json/PMC7774577.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14601</th>\n",
       "      <td>embv1el9</td>\n",
       "      <td>2b4690b639d799f33bbf7cb9eec5522878310594</td>\n",
       "      <td>PMC</td>\n",
       "      <td>Accuracy of a battery-powered portable capnome...</td>\n",
       "      <td>10.1007/s10877-020-00638-0</td>\n",
       "      <td>PMC7775636</td>\n",
       "      <td>33385261</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>Carbon dioxide measurement is useful for confi...</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>Hirakawa, Eiji; Ibara, Satoshi</td>\n",
       "      <td>J Clin Monit Comput</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/2b4690b639d799f33bbf7...</td>\n",
       "      <td>document_parses/pmc_json/PMC7775636.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14602</th>\n",
       "      <td>8dcgzyst</td>\n",
       "      <td>83a841d74d4f6bf8bffb818a5cd6b7fa85d63ff2</td>\n",
       "      <td>PMC</td>\n",
       "      <td>The influence of diabetes on postoperative com...</td>\n",
       "      <td>10.1007/s10151-020-02373-9</td>\n",
       "      <td>PMC7775741</td>\n",
       "      <td>33386511</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>BACKGROUND: Diabetes mellitus has been commonl...</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>Tan, D. J. H.; Yaow, C. Y. L.; Mok, H. T.; Ng,...</td>\n",
       "      <td>Tech Coloproctol</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/83a841d74d4f6bf8bffb8...</td>\n",
       "      <td>document_parses/pmc_json/PMC7775741.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14603</th>\n",
       "      <td>sgtvecj8</td>\n",
       "      <td>7dd8097ba18590caa431084fa22b6580702d5788</td>\n",
       "      <td>PMC</td>\n",
       "      <td>In vivo delivery of a multiepitope peptide and...</td>\n",
       "      <td>10.1007/s10529-020-03060-3</td>\n",
       "      <td>PMC7775797</td>\n",
       "      <td>33386500</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>OBJECTIVES: A potent HIV vaccine should overco...</td>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>Davoodi, Saba; Bolhassani, Azam; Namazi, Fatemeh</td>\n",
       "      <td>Biotechnol Lett</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/7dd8097ba18590caa4310...</td>\n",
       "      <td>document_parses/pmc_json/PMC7775797.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14604</th>\n",
       "      <td>3vtq6d8f</td>\n",
       "      <td>9bd542f156f108ef170a0eec832161d0edb3d2f8</td>\n",
       "      <td>PMC</td>\n",
       "      <td>Relationship Between Compassion Fatigue in Nur...</td>\n",
       "      <td>10.1007/s10943-020-01142-0</td>\n",
       "      <td>PMC7775832</td>\n",
       "      <td>33386572</td>\n",
       "      <td>no-cc</td>\n",
       "      <td>This research was conducted in order to examin...</td>\n",
       "      <td>2021-01-02</td>\n",
       "      <td>Aslan, Hakime; Erci, Behice; Pekince, Hatice</td>\n",
       "      <td>J Relig Health</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/9bd542f156f108ef170a0...</td>\n",
       "      <td>document_parses/pmc_json/PMC7775832.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764526</th>\n",
       "      <td>rxz7w4r5</td>\n",
       "      <td>5dd2a11964526f9ab278561fa4fd2201e0a99085</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>Therapeutic Targeting of the Complement System...</td>\n",
       "      <td>10.1124/pharmrev.120.000072</td>\n",
       "      <td>PMC7956994</td>\n",
       "      <td>3.3688e+07</td>\n",
       "      <td>cc-by-nc</td>\n",
       "      <td>The complement system was discovered at the en...</td>\n",
       "      <td>2021-04-27</td>\n",
       "      <td>Garred, Peter; Tenner, Andrea J.; Mollnes, Tom E.</td>\n",
       "      <td>Pharmacol Rev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/5dd2a11964526f9ab2785...</td>\n",
       "      <td>document_parses/pmc_json/PMC7956994.xml.json</td>\n",
       "      <td>https://doi.org/10.1124/pharmrev.120.000072; h...</td>\n",
       "      <td>232169901.0</td>\n",
       "      <td>2021-04-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764528</th>\n",
       "      <td>ek63ruc2</td>\n",
       "      <td>ca2ff01b250ecb180a9a5ee54031dcaf4ad78f51</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>Dysregulation of Leukocyte Trafficking in Type...</td>\n",
       "      <td>10.3389/fcell.2021.624184</td>\n",
       "      <td>PMC7937619</td>\n",
       "      <td>3.3693e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>Type 2 Diabetes Mellitus (T2DM) is a chronic i...</td>\n",
       "      <td>2021-02-22</td>\n",
       "      <td>Pezhman, Laleh; Tahrani, Abd; Chimen, Myriam</td>\n",
       "      <td>Front Cell Dev Biol</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/ca2ff01b250ecb180a9a5...</td>\n",
       "      <td>document_parses/pmc_json/PMC7937619.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/33692997/;...</td>\n",
       "      <td>231981196.0</td>\n",
       "      <td>2021-02-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764538</th>\n",
       "      <td>1jpcr63l</td>\n",
       "      <td>075844e8ba8d7797189f57f5114a9a67c8c8999c</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>The application of drones for mosquito larval ...</td>\n",
       "      <td>10.1186/s12936-021-03759-2</td>\n",
       "      <td>PMC8165685</td>\n",
       "      <td>3.40591e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>BACKGROUND: Spatio-temporal trends in mosquito...</td>\n",
       "      <td>2021-05-31</td>\n",
       "      <td>Stanton, Michelle C.; Kalonde, Patrick; Zember...</td>\n",
       "      <td>Malar J</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/075844e8ba8d7797189f5...</td>\n",
       "      <td>document_parses/pmc_json/PMC8165685.xml.json</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/34059053/;...</td>\n",
       "      <td>235260179.0</td>\n",
       "      <td>2021-05-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764542</th>\n",
       "      <td>4p5r3410</td>\n",
       "      <td>b5881e8c6badd610b7a8f8b59ac207dbd76bb47b</td>\n",
       "      <td>Medline; PMC</td>\n",
       "      <td>Development of Nanobodies Targeting Peste des ...</td>\n",
       "      <td>10.3390/ani11082206</td>\n",
       "      <td>PMC8388416</td>\n",
       "      <td>3.44387e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>SIMPLE SUMMARY: Peste des petits ruminants vir...</td>\n",
       "      <td>2021-07-26</td>\n",
       "      <td>Kinimi, Edson; Muyldermans, Serge; Vincke, Céc...</td>\n",
       "      <td>Animals (Basel)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/b5881e8c6badd610b7a8f...</td>\n",
       "      <td>document_parses/pmc_json/PMC8388416.xml.json</td>\n",
       "      <td>https://doi.org/10.3390/ani11082206; https://w...</td>\n",
       "      <td>237321137.0</td>\n",
       "      <td>2021-07-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764545</th>\n",
       "      <td>pgj09tww</td>\n",
       "      <td>96536cef98966bb717a21174d4d4848e9e10f749</td>\n",
       "      <td>Medline; PMC; WHO</td>\n",
       "      <td>Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...</td>\n",
       "      <td>10.1007/s11831-021-09638-y</td>\n",
       "      <td>PMC8381867</td>\n",
       "      <td>3.44566e+07</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>The timing and sequence of safe campus reopeni...</td>\n",
       "      <td>2021-08-23</td>\n",
       "      <td>Linka, Kevin; Peirlinck, Mathias; Schäfer, Ame...</td>\n",
       "      <td>Arch Comput Methods Eng</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>document_parses/pdf_json/96536cef98966bb717a21...</td>\n",
       "      <td>document_parses/pmc_json/PMC8381867.xml.json</td>\n",
       "      <td>https://doi.org/10.1007/s11831-021-09638-y; ht...</td>\n",
       "      <td>237279499.0</td>\n",
       "      <td>2021-08-23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109528 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        cord_uid                                       sha           source_x  \\\n",
       "14597   aoiambb8  3798d3bcb09df782968320b8ad39f28a1bed48e6                PMC   \n",
       "14601   embv1el9  2b4690b639d799f33bbf7cb9eec5522878310594                PMC   \n",
       "14602   8dcgzyst  83a841d74d4f6bf8bffb818a5cd6b7fa85d63ff2                PMC   \n",
       "14603   sgtvecj8  7dd8097ba18590caa431084fa22b6580702d5788                PMC   \n",
       "14604   3vtq6d8f  9bd542f156f108ef170a0eec832161d0edb3d2f8                PMC   \n",
       "...          ...                                       ...                ...   \n",
       "764526  rxz7w4r5  5dd2a11964526f9ab278561fa4fd2201e0a99085       Medline; PMC   \n",
       "764528  ek63ruc2  ca2ff01b250ecb180a9a5ee54031dcaf4ad78f51       Medline; PMC   \n",
       "764538  1jpcr63l  075844e8ba8d7797189f57f5114a9a67c8c8999c       Medline; PMC   \n",
       "764542  4p5r3410  b5881e8c6badd610b7a8f8b59ac207dbd76bb47b       Medline; PMC   \n",
       "764545  pgj09tww  96536cef98966bb717a21174d4d4848e9e10f749  Medline; PMC; WHO   \n",
       "\n",
       "                                                    title  \\\n",
       "14597   Etiology of Severe Acute Respiratory Infection...   \n",
       "14601   Accuracy of a battery-powered portable capnome...   \n",
       "14602   The influence of diabetes on postoperative com...   \n",
       "14603   In vivo delivery of a multiepitope peptide and...   \n",
       "14604   Relationship Between Compassion Fatigue in Nur...   \n",
       "...                                                   ...   \n",
       "764526  Therapeutic Targeting of the Complement System...   \n",
       "764528  Dysregulation of Leukocyte Trafficking in Type...   \n",
       "764538  The application of drones for mosquito larval ...   \n",
       "764542  Development of Nanobodies Targeting Peste des ...   \n",
       "764545  Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...   \n",
       "\n",
       "                                doi       pmcid    pubmed_id   license  \\\n",
       "14597        10.3201/eid2701.201347  PMC7774577     33350930     no-cc   \n",
       "14601    10.1007/s10877-020-00638-0  PMC7775636     33385261     no-cc   \n",
       "14602    10.1007/s10151-020-02373-9  PMC7775741     33386511     no-cc   \n",
       "14603    10.1007/s10529-020-03060-3  PMC7775797     33386500     no-cc   \n",
       "14604    10.1007/s10943-020-01142-0  PMC7775832     33386572     no-cc   \n",
       "...                             ...         ...          ...       ...   \n",
       "764526  10.1124/pharmrev.120.000072  PMC7956994   3.3688e+07  cc-by-nc   \n",
       "764528    10.3389/fcell.2021.624184  PMC7937619   3.3693e+07     cc-by   \n",
       "764538   10.1186/s12936-021-03759-2  PMC8165685  3.40591e+07     cc-by   \n",
       "764542          10.3390/ani11082206  PMC8388416  3.44387e+07     cc-by   \n",
       "764545   10.1007/s11831-021-09638-y  PMC8381867  3.44566e+07     cc-by   \n",
       "\n",
       "                                                 abstract publish_time  \\\n",
       "14597   In April 2017, surveillance detected a surge i...   2021-01-27   \n",
       "14601   Carbon dioxide measurement is useful for confi...   2021-01-01   \n",
       "14602   BACKGROUND: Diabetes mellitus has been commonl...   2021-01-01   \n",
       "14603   OBJECTIVES: A potent HIV vaccine should overco...   2021-01-01   \n",
       "14604   This research was conducted in order to examin...   2021-01-02   \n",
       "...                                                   ...          ...   \n",
       "764526  The complement system was discovered at the en...   2021-04-27   \n",
       "764528  Type 2 Diabetes Mellitus (T2DM) is a chronic i...   2021-02-22   \n",
       "764538  BACKGROUND: Spatio-temporal trends in mosquito...   2021-05-31   \n",
       "764542  SIMPLE SUMMARY: Peste des petits ruminants vir...   2021-07-26   \n",
       "764545  The timing and sequence of safe campus reopeni...   2021-08-23   \n",
       "\n",
       "                                                  authors  \\\n",
       "14597   Rahaman, Md R.; Alroy, Karen A.; Van Beneden, ...   \n",
       "14601                      Hirakawa, Eiji; Ibara, Satoshi   \n",
       "14602   Tan, D. J. H.; Yaow, C. Y. L.; Mok, H. T.; Ng,...   \n",
       "14603    Davoodi, Saba; Bolhassani, Azam; Namazi, Fatemeh   \n",
       "14604        Aslan, Hakime; Erci, Behice; Pekince, Hatice   \n",
       "...                                                   ...   \n",
       "764526  Garred, Peter; Tenner, Andrea J.; Mollnes, Tom E.   \n",
       "764528       Pezhman, Laleh; Tahrani, Abd; Chimen, Myriam   \n",
       "764538  Stanton, Michelle C.; Kalonde, Patrick; Zember...   \n",
       "764542  Kinimi, Edson; Muyldermans, Serge; Vincke, Céc...   \n",
       "764545  Linka, Kevin; Peirlinck, Mathias; Schäfer, Ame...   \n",
       "\n",
       "                        journal  mag_id who_covidence_id arxiv_id  \\\n",
       "14597          Emerg Infect Dis     NaN              NaN      NaN   \n",
       "14601       J Clin Monit Comput     NaN              NaN      NaN   \n",
       "14602          Tech Coloproctol     NaN              NaN      NaN   \n",
       "14603           Biotechnol Lett     NaN              NaN      NaN   \n",
       "14604            J Relig Health     NaN              NaN      NaN   \n",
       "...                         ...     ...              ...      ...   \n",
       "764526            Pharmacol Rev     NaN              NaN      NaN   \n",
       "764528      Front Cell Dev Biol     NaN              NaN      NaN   \n",
       "764538                  Malar J     NaN              NaN      NaN   \n",
       "764542          Animals (Basel)     NaN              NaN      NaN   \n",
       "764545  Arch Comput Methods Eng     NaN              NaN      NaN   \n",
       "\n",
       "                                           pdf_json_files  \\\n",
       "14597   document_parses/pdf_json/3798d3bcb09df78296832...   \n",
       "14601   document_parses/pdf_json/2b4690b639d799f33bbf7...   \n",
       "14602   document_parses/pdf_json/83a841d74d4f6bf8bffb8...   \n",
       "14603   document_parses/pdf_json/7dd8097ba18590caa4310...   \n",
       "14604   document_parses/pdf_json/9bd542f156f108ef170a0...   \n",
       "...                                                   ...   \n",
       "764526  document_parses/pdf_json/5dd2a11964526f9ab2785...   \n",
       "764528  document_parses/pdf_json/ca2ff01b250ecb180a9a5...   \n",
       "764538  document_parses/pdf_json/075844e8ba8d7797189f5...   \n",
       "764542  document_parses/pdf_json/b5881e8c6badd610b7a8f...   \n",
       "764545  document_parses/pdf_json/96536cef98966bb717a21...   \n",
       "\n",
       "                                      pmc_json_files  \\\n",
       "14597   document_parses/pmc_json/PMC7774577.xml.json   \n",
       "14601   document_parses/pmc_json/PMC7775636.xml.json   \n",
       "14602   document_parses/pmc_json/PMC7775741.xml.json   \n",
       "14603   document_parses/pmc_json/PMC7775797.xml.json   \n",
       "14604   document_parses/pmc_json/PMC7775832.xml.json   \n",
       "...                                              ...   \n",
       "764526  document_parses/pmc_json/PMC7956994.xml.json   \n",
       "764528  document_parses/pmc_json/PMC7937619.xml.json   \n",
       "764538  document_parses/pmc_json/PMC8165685.xml.json   \n",
       "764542  document_parses/pmc_json/PMC8388416.xml.json   \n",
       "764545  document_parses/pmc_json/PMC8381867.xml.json   \n",
       "\n",
       "                                                      url        s2_id  \\\n",
       "14597   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...          NaN   \n",
       "14601   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...          NaN   \n",
       "14602   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...          NaN   \n",
       "14603   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...          NaN   \n",
       "14604   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7...          NaN   \n",
       "...                                                   ...          ...   \n",
       "764526  https://doi.org/10.1124/pharmrev.120.000072; h...  232169901.0   \n",
       "764528  https://www.ncbi.nlm.nih.gov/pubmed/33692997/;...  231981196.0   \n",
       "764538  https://www.ncbi.nlm.nih.gov/pubmed/34059053/;...  235260179.0   \n",
       "764542  https://doi.org/10.3390/ani11082206; https://w...  237321137.0   \n",
       "764545  https://doi.org/10.1007/s11831-021-09638-y; ht...  237279499.0   \n",
       "\n",
       "             date  \n",
       "14597  2021-01-27  \n",
       "14601  2021-01-01  \n",
       "14602  2021-01-01  \n",
       "14603  2021-01-01  \n",
       "14604  2021-01-02  \n",
       "...           ...  \n",
       "764526 2021-04-27  \n",
       "764528 2021-02-22  \n",
       "764538 2021-05-31  \n",
       "764542 2021-07-26  \n",
       "764545 2021-08-23  \n",
       "\n",
       "[109528 rows x 20 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(metadata) - len(metadata[metadata['pdf_json_files'].notna() | metadata['pmc_json_files'].notna() | metadata['abstract'].notna()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 109528/109528 [01:34<00:00, 1153.94it/s]\n"
     ]
    }
   ],
   "source": [
    "pmc_count = 0\n",
    "pdf_count = 0\n",
    "results_list = []\n",
    "for index, data in tqdm(metadata.iterrows(), total=len(metadata)):\n",
    "    data_dict = dict()\n",
    "    try:\n",
    "        if pd.isna(data['pdf_json_files']):\n",
    "            pmc_count += 1\n",
    "            file_loc = '../CORD19/' + data['pmc_json_files']\n",
    "            with open(file_loc) as json_file:\n",
    "                target_json = json.load(json_file)\n",
    "                data_dict['doc_id'] = target_json['paper_id']\n",
    "                data_dict['title'] = target_json['metadata']['title']\n",
    "                data_dict['cord_uid'] = data['cord_uid']\n",
    "                abstract_section = str()\n",
    "                data_dict['abstract'] = data['abstract']\n",
    "                full_text_section = str()\n",
    "                for element in target_json['body_text']:\n",
    "                    full_text_section += element['text'] + ' '\n",
    "                data_dict['full_text'] = full_text_section\n",
    "        else:\n",
    "            pdf_count += 1\n",
    "            # PDF jsons file contains multiple docs separated by \";\"\n",
    "            file_loc = '../CORD19/' + data['pdf_json_files']\n",
    "            file_loc = file_loc.split('; ')[0]\n",
    "            with open(file_loc) as json_file:\n",
    "                target_json = json.load(json_file)\n",
    "                data_dict['doc_id'] = target_json['paper_id']\n",
    "                data_dict['title'] = target_json['metadata']['title']\n",
    "                data_dict['cord_uid'] = data['cord_uid']\n",
    "                abstract_section = str()\n",
    "                if target_json['abstract'] != '':\n",
    "                    for element in target_json['abstract']:\n",
    "                        abstract_section += element['text'] + ' '\n",
    "                else:\n",
    "                    abstract_section = data['abstract']\n",
    "                data_dict['abstract'] = abstract_section\n",
    "                full_text_section = str()\n",
    "                for element in target_json['body_text']:\n",
    "                    full_text_section += element['text'] + ' '\n",
    "                data_dict['full_text'] = full_text_section\n",
    "        results_list.append(data_dict)\n",
    "    except Exception as msg:\n",
    "        print('Error at: ' + data['pdf_json_files'])\n",
    "        print(msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total PMC records: 5247\n",
      "Total PDF records: 104281\n"
     ]
    }
   ],
   "source": [
    "print('Total PMC records: {}'.format(pmc_count))\n",
    "print('Total PDF records: {}'.format(pdf_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(results_list)\n",
    "df_results.to_csv('total_corpus.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cord_uid</th>\n",
       "      <th>abstract</th>\n",
       "      <th>full_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>109523</th>\n",
       "      <td>5dd2a11964526f9ab278561fa4fd2201e0a99085</td>\n",
       "      <td>Therapeutic Targeting of the Complement System...</td>\n",
       "      <td>rxz7w4r5</td>\n",
       "      <td></td>\n",
       "      <td>ABBREVIATIONS: AAV, anti-neutrophilic cytoplas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109524</th>\n",
       "      <td>ca2ff01b250ecb180a9a5ee54031dcaf4ad78f51</td>\n",
       "      <td>Dysregulation of Leukocyte Trafficking in Type...</td>\n",
       "      <td>ek63ruc2</td>\n",
       "      <td>Type 2 Diabetes Mellitus (T2DM) is a chronic i...</td>\n",
       "      <td>Inflammation is a protective response against ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109525</th>\n",
       "      <td>075844e8ba8d7797189f57f5114a9a67c8c8999c</td>\n",
       "      <td>The application of drones for mosquito larval ...</td>\n",
       "      <td>1jpcr63l</td>\n",
       "      <td>Background: Spatio-temporal trends in mosquito...</td>\n",
       "      <td>Malaria cases in Africa have reduced by over h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109526</th>\n",
       "      <td>b5881e8c6badd610b7a8f8b59ac207dbd76bb47b</td>\n",
       "      <td>Development of Nanobodies Targeting Peste des ...</td>\n",
       "      <td>4p5r3410</td>\n",
       "      <td>Citation: Kinimi, E.; Muyldermans, S.; Vincke,...</td>\n",
       "      <td>Peste des petits ruminants virus (PPRV) causes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109527</th>\n",
       "      <td>96536cef98966bb717a21174d4d4848e9e10f749</td>\n",
       "      <td>Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...</td>\n",
       "      <td>pgj09tww</td>\n",
       "      <td>The timing and sequence of safe campus reopeni...</td>\n",
       "      <td>More than half a million cases of COVID-19 hav...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          doc_id  \\\n",
       "109523  5dd2a11964526f9ab278561fa4fd2201e0a99085   \n",
       "109524  ca2ff01b250ecb180a9a5ee54031dcaf4ad78f51   \n",
       "109525  075844e8ba8d7797189f57f5114a9a67c8c8999c   \n",
       "109526  b5881e8c6badd610b7a8f8b59ac207dbd76bb47b   \n",
       "109527  96536cef98966bb717a21174d4d4848e9e10f749   \n",
       "\n",
       "                                                    title  cord_uid  \\\n",
       "109523  Therapeutic Targeting of the Complement System...  rxz7w4r5   \n",
       "109524  Dysregulation of Leukocyte Trafficking in Type...  ek63ruc2   \n",
       "109525  The application of drones for mosquito larval ...  1jpcr63l   \n",
       "109526  Development of Nanobodies Targeting Peste des ...  4p5r3410   \n",
       "109527  Effects of B.1.1.7 and B.1.351 on COVID-19 Dyn...  pgj09tww   \n",
       "\n",
       "                                                 abstract  \\\n",
       "109523                                                      \n",
       "109524  Type 2 Diabetes Mellitus (T2DM) is a chronic i...   \n",
       "109525  Background: Spatio-temporal trends in mosquito...   \n",
       "109526  Citation: Kinimi, E.; Muyldermans, S.; Vincke,...   \n",
       "109527  The timing and sequence of safe campus reopeni...   \n",
       "\n",
       "                                                full_text  \n",
       "109523  ABBREVIATIONS: AAV, anti-neutrophilic cytoplas...  \n",
       "109524  Inflammation is a protective response against ...  \n",
       "109525  Malaria cases in Africa have reduced by over h...  \n",
       "109526  Peste des petits ruminants virus (PPRV) causes...  \n",
       "109527  More than half a million cases of COVID-19 hav...  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cord_uid</th>\n",
       "      <th>abstract</th>\n",
       "      <th>full_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>109528</td>\n",
       "      <td>109528</td>\n",
       "      <td>109528</td>\n",
       "      <td>107141</td>\n",
       "      <td>109528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>109509</td>\n",
       "      <td>96824</td>\n",
       "      <td>109523</td>\n",
       "      <td>77961</td>\n",
       "      <td>109229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>31bc0fb718edaab9e33f678909710f62c40abebc</td>\n",
       "      <td></td>\n",
       "      <td>6v6mdrk0</td>\n",
       "      <td></td>\n",
       "      <td>Nil. There are no conflicts of interest.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>3</td>\n",
       "      <td>9064</td>\n",
       "      <td>2</td>\n",
       "      <td>28660</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          doc_id   title  cord_uid abstract  \\\n",
       "count                                     109528  109528    109528   107141   \n",
       "unique                                    109509   96824    109523    77961   \n",
       "top     31bc0fb718edaab9e33f678909710f62c40abebc          6v6mdrk0            \n",
       "freq                                           3    9064         2    28660   \n",
       "\n",
       "                                        full_text  \n",
       "count                                      109528  \n",
       "unique                                     109229  \n",
       "top     Nil. There are no conflicts of interest.   \n",
       "freq                                          149  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}