idr-study-publications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count publications in the IDR\n",
    "\n",
    "This notebook uses the IDR OMERO.web API to fetch the top-level study metadata for all published studies, and fetches Pubmed metadata for each study that contains a Pubmed ID.\n",
    "\n",
    "This is used to show the journals and number of studies associated with that journal in the IDR."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "import requests\n",
    "from time import sleep\n",
    "from xml.dom.minidom import parseString\n",
    "\n",
    "IDR_BASE_URL = 'https://idr.openmicroscopy.org'\n",
    "INDEX_PAGE = f'{IDR_BASE_URL}/webclient/?experimenter=-1'\n",
    "MAP_URL = '{base}/webclient/api/annotations/?type=map&{type}={id}'\n",
    "PUBMED_BATCH_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={pmids}&retmode=xml'\n",
    "CROSSREF_URL = 'https://api.crossref.org/works/{doi}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get all studies (projects and screens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 62 screens 62 projects\n"
     ]
    }
   ],
   "source": [
    "screens = requests.get(f'{IDR_BASE_URL}/api/v0/m/screens/').json()\n",
    "projects = requests.get(f'{IDR_BASE_URL}/api/v0/m/projects/').json()\n",
    "print(f\"Found {screens['meta']['totalCount']} screens {projects['meta']['totalCount']} projects\")\n",
    "assert (screens['meta']['totalCount'] <= screens['meta']['limit']), 'Paging required'\n",
    "assert (projects['meta']['totalCount'] <= projects['meta']['limit']), 'Paging required'\n",
    "\n",
    "studies = {\n",
    "    'screen': screens['data'],\n",
    "    'project': projects['data'],\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get study map annotations, we're interested in the PubMed IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PublicationIds:\n",
    "    def __init__(self, *, pubmed=None, doi=None):\n",
    "        self.pubmed = pubmed\n",
    "        self.doi = doi\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f'PublicationIds(pubmed={self.pubmed}, doi={self.doi}'\n",
    "\n",
    "study_pubids = defaultdict(PublicationIds)\n",
    "names = set()\n",
    "\n",
    "for (study_type, container) in studies.items():\n",
    "    for c in container:\n",
    "        url = MAP_URL.format(base=IDR_BASE_URL, type=study_type, id=c['@id'])\n",
    "        annotations = requests.get(url).json()['annotations']\n",
    "        for a in annotations:\n",
    "            if a['ns'] == 'idr.openmicroscopy.org/study/info':\n",
    "                m = dict(a['values'])\n",
    "                name = c['Name'].split('/')[0]\n",
    "                names.add(name)\n",
    "                try:\n",
    "                    pubmedid = m['PubMed ID'].split()[0]\n",
    "                    if study_pubids[name].pubmed:\n",
    "                        assert study_pubids[name].pubmed == pubmedid\n",
    "                    else:\n",
    "                        study_pubids[name].pubmed = pubmedid                        \n",
    "                except KeyError:\n",
    "                    pass\n",
    "                try:\n",
    "                    doi = m['Publication DOI'].split()[0]\n",
    "                    if study_pubids[name].doi:\n",
    "                        assert study_pubids[name].doi == doi\n",
    "                    else:\n",
    "                        study_pubids[name].doi = doi                        \n",
    "                except KeyError:\n",
    "                    pass\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking up 68 Pubmed IDs 1 DOIs\n",
      "\n",
      "Missing Pubmed ID and DOI:\n",
      "\tidr0011-ledesmafernandez-dad4\n",
      "\tidr0018-neff-histopathology\n",
      "\tidr0085-walsh-mfhrem\n",
      "\tidr0091-julou-lacinduction\n",
      "\tidr0092-ostrop-organoid\n",
      "\tidr0094-ellinger-sarscov2\n"
     ]
    }
   ],
   "source": [
    "pmids = []\n",
    "dois = []\n",
    "\n",
    "for (name, pids) in sorted(study_pubids.items()):\n",
    "#     print(name, pids)\n",
    "    if pids.pubmed:\n",
    "        pmids.append(pids.pubmed)\n",
    "    else:\n",
    "        dois.append(pids.doi)\n",
    "\n",
    "print(f'Looking up {len(pmids)} Pubmed IDs {len(dois)} DOIs')\n",
    "\n",
    "missing = sorted(names.difference(study_pubids.keys()))\n",
    "if missing:\n",
    "    print('\\nMissing Pubmed ID and DOI:\\n\\t' + '\\n\\t'.join(missing))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now fetch the Pubmed metadata using a single batch query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(PUBMED_BATCH_URL.format(pmids=','.join(pmids)))\n",
    "r.raise_for_status()\n",
    "\n",
    "rxml = parseString(r.text)\n",
    "journalxml = rxml.getElementsByTagName('Journal')\n",
    "assert len(journalxml) == len(pmids)\n",
    "journal_titles = [j.getElementsByTagName('Title')[0].firstChild.data for j in journalxml]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If a journal has a DOI but not a Pubmed ID lookup the journal using Crossref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "doi_journal_titles = []\n",
    "for doi in dois:\n",
    "    sleep(1)\n",
    "    r = requests.get(CROSSREF_URL.format(doi=doi))\n",
    "    r.raise_for_status()\n",
    "    crossref = r.json()\n",
    "    doi_journal_titles.append(crossref['message']['container-title'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally list the journals and the number of studies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             The Journal of cell biology   7\n",
      "                                   eLife   7\n",
      "                   Nature communications   6\n",
      "               Molecular systems biology   5\n",
      "                                PloS one   4\n",
      "                Science (New York, N.Y.)   4\n",
      "                                  Nature   3\n",
      "                          Nature methods   3\n",
      "                         Scientific data   3\n",
      "                      Scientific reports   3\n",
      "                                    Cell   2\n",
      "                            Cell systems   2\n",
      "                     Nature cell biology   2\n",
      "                        BMC cell biology   1\n",
      "                        Cancer discovery   1\n",
      "                            Cell reports   1\n",
      "                    Current biology : CB   1\n",
      "        Development (Cambridge, England)   1\n",
      "                      Developmental cell   1\n",
      "                             GigaScience   1\n",
      "                      Journal of anatomy   1\n",
      "                   Journal of proteomics   1\n",
      "           Molecular biology of the cell   1\n",
      "                           Nature Cancer   1\n",
      "                            PLoS biology   1\n",
      "              PLoS computational biology   1\n",
      "                           PLoS genetics   1\n",
      "Proceedings of the National Academy of Sciences of the United States of America   1\n",
      "                        Science advances   1\n",
      "                        The EMBO journal   1\n",
      "    The Journal of experimental medicine   1\n"
     ]
    }
   ],
   "source": [
    "journals = Counter(journal_titles + doi_journal_titles)\n",
    "journals = sorted(journals.items(), key=lambda x: (-x[1], x[0]))\n",
    "\n",
    "for j in journals:\n",
    "    print(f'{j[0]:>40} {j[1]:3}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}