request_ncbi_text_for_genes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e33a685",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import html2text\n",
    "import mygene\n",
    "import json\n",
    "import pickle\n",
    "mg = mygene.MyGeneInfo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "636944e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "parts_to_remove = [\n",
    "    \"##  Summary\\n\",\n",
    "    \"NEW\",\n",
    "    'Try the newGene table',\n",
    "    'Try the newTranscript table',\n",
    "    '**',\n",
    "    \"\\nGo to the top of the page Help\\n\"\n",
    "]\n",
    "\n",
    "def rough_text_from_gene_name(gene_number):\n",
    "    \n",
    "    # get url\n",
    "    url = f\"https://www.ncbi.nlm.nih.gov/gene/{gene_number}\"\n",
    "    # Send a GET request to the URL\n",
    "    summary_text = ''\n",
    "    soup = None\n",
    "    try:\n",
    "        response = requests.get(url, timeout=30)\n",
    "    except requests.exceptions.Timeout:\n",
    "        print('time out')\n",
    "        return((summary_text,soup))\n",
    "    # Check if the request was successful (status code 200)\n",
    "    if response.status_code == 200:\n",
    "        # Parse the HTML content of the page\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "        # Find the \"summary\" tab content by inspecting the page's structure\n",
    "        summary_tab = soup.find('div', {'class': 'rprt-section gene-summary'})\n",
    "\n",
    "        # Check if the \"summary\" tab content is found\n",
    "        if summary_tab:\n",
    "            # Convert the HTML to plain text using html2text\n",
    "            html_to_text = html2text.HTML2Text()\n",
    "            html_to_text.ignore_links = True  # Ignore hyperlinks\n",
    "\n",
    "            # Extract the plain text from the \"summary\" tab\n",
    "            summary_text = html_to_text.handle(str(summary_tab))\n",
    "            # Remove the specified parts from the original text\n",
    "            for part in parts_to_remove:\n",
    "                summary_text = summary_text.replace(part, ' ')\n",
    "                # Replace '\\n' with a space\n",
    "            summary_text = summary_text.replace('\\n', ' ')\n",
    "\n",
    "            # Reduce multiple spaces into one space\n",
    "            summary_text = ' '.join(summary_text.split())\n",
    "            # Print or save the extracted text\n",
    "        else:\n",
    "            print(\"Summary tab not found on the page.\")\n",
    "    else:\n",
    "        print(f\"Failed to retrieve the webpage. Status code: {response.status_code}\")\n",
    "    return((summary_text,soup))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b22b8485",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using Gene CD24 as an example\n",
    "cd_24_name = mg.querymany('CD24', scopes='symbol', species='human')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d8b506e",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_name_to_tax_id = {}\n",
    "for result in cd_24_name:\n",
    "    if \"_id\" in result and \"query\" in result:\n",
    "        gene_name_to_tax_id[result['symbol']] = result['_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a80980bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_name_to_tax_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b2a50fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/yiquntchen/Downloads/human/vocab.json', 'rb') as handle:\n",
    "    vocab_gene = json.load(handle)\n",
    "vocab_gene_list = list(vocab_gene.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2d705d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_name_to_summary_page = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4e98122",
   "metadata": {},
   "outputs": [],
   "source": [
    "for gene_name, page_id in sorted(gene_name_to_tax_id.items()):\n",
    "    if gene_name not in gene_name_to_summary_page:\n",
    "        print('gene_name',gene_name)\n",
    "        parsed_text, unparsed_html = rough_text_from_gene_name(page_id)\n",
    "        gene_name_to_summary_page[gene_name] = parsed_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ab7ea98",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_name_to_summary_page"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdb9a9ac",
   "metadata": {},
   "source": [
    "### Experiment with Varied Gene Sets Tailored to Your Needs\n",
    "#### For any specific gene name, mygene can be utilized to translate it into page IDs. We've illustrated an example using gene vocabularies in scGPT and Geneformer. Download links for these files are available in the repository's README."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d609d3ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load genes used in GenePT\n",
    "with open(f\"{data_dir}/vocab.json', 'rb') as handle:\n",
    "    vocab_gene = json.load(handle)\n",
    "vocab_gene_list = list(vocab_gene.keys())\n",
    "\n",
    "# load genes used in Geneformer\n",
    "with open(f\"{data_dir}/token_dictionary.pkl\", 'rb') as handle:\n",
    "    token_dictionary = pickle.load(handle)\n",
    "\n",
    "# example query to convert gene IDs into page ids for NCBI \n",
    "vocab_gene_list_results = mg.querymany(sorted(vocab_gene_list), scopes='symbol', species='human')\n",
    "token_dictionary_results = mg.querymany(sorted(token_dictionary.keys()), fields=\"symbol\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}