scxa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "from _pathlib import Path\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% Do some imports\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "scxa_url = 'https://www.ebi.ac.uk/gxa/sc/json/experiments'\n",
    "response = requests.get(scxa_url)\n",
    "projects = response.json()['experiments']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def project_url(accession: str):\n",
    "    return f'https://www.ebi.ac.uk/gxa/sc/experiments/{accession}/results/tsne'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# Don't actually need this... Assays is a synonym for cells. \n",
    "def get_cell_count(accession: str) -> int:\n",
    "    response_ = requests.get(project_url(accession))\n",
    "    soup = BeautifulSoup(response_.content, 'html.parser')\n",
    "    h3 = soup.find(id='goto-experiment')\n",
    "    cell_count_text = list(h3.next_siblings)[3].text\n",
    "    _, cell_count_text = cell_count_text.split(':')\n",
    "    return int(cell_count_text.strip().replace(',', ''))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "def cell_count(projects_):\n",
    "    return sum(p['numberOfAssays'] for p in projects_)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "1028590"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 6
    }
   ],
   "source": [
    "cell_count(projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "177678"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 7
    }
   ],
   "source": [
    "hca_projects = [p for p in projects if 'Human Cell Atlas' in p['experimentProjects']]\n",
    "cell_count(hca_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "198047"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 8
    }
   ],
   "source": [
    "geo_projects = [p for p in projects if 'GEO' in p['experimentAccession']]\n",
    "cell_count(geo_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "def geo_accession(project):\n",
    "    scxa_accession = project['experimentAccession']\n",
    "    prefix = 'E-GEOD-'\n",
    "    assert scxa_accession.startswith(prefix)\n",
    "    return 'GSE' + scxa_accession[len(prefix):]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "for geo in geo_projects:\n",
    "    geo['geo_accession'] = geo_accession(geo)\n",
    "\n",
    "geo_accessions = [geo['geo_accession'] for geo in geo_projects]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "161359"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 11
    }
   ],
   "source": [
    "loaded_geo_accessions = {\n",
    "'GSE100618', 'GSE102580', 'GSE102596', 'GSE103275', 'GSE103354', 'GSE106273', 'GSE106540', 'GSE107585', 'GSE107618',\n",
    "'GSE107746', 'GSE107909', 'GSE108041', 'GSE108291', 'GSE109488', 'GSE109822', 'GSE109979', 'GSE110154', 'GSE110499',\n",
    "'GSE111586', 'GSE111727', 'GSE113197', 'GSE114374', 'GSE114396', 'GSE114557', 'GSE114802', 'GSE115469', 'GSE116237',\n",
    "'GSE116470', 'GSE117089', 'GSE117498', 'GSE118127', 'GSE124472', 'GSE124494', 'GSE126836', 'GSE127969', 'GSE128639',\n",
    "'GSE129798', 'GSE130430', 'GSE130473', 'GSE130606', 'GSE130636', 'GSE131181', 'GSE131685', 'GSE131736', 'GSE132040',\n",
    "'GSE132044', 'GSE132566', 'GSE132802', 'GSE134881', 'GSE135889', 'GSE36552', 'GSE44183', 'GSE57872', 'GSE67835',\n",
    "'GSE70580', 'GSE73727', 'GSE75140', 'GSE75367', 'GSE75478', 'GSE75659', 'GSE75688', 'GSE76312', 'GSE76381',\n",
    "'GSE81383', 'GSE81547', 'GSE81608', 'GSE81904', 'GSE81905', 'GSE83139', 'GSE84133', 'GSE84147', 'GSE84465',\n",
    "'GSE86146', 'GSE86469', 'GSE86473', 'GSE89232', 'GSE89322', 'GSE90806', 'GSE92280', 'GSE93374', 'GSE93593',\n",
    "'GSE94820', 'GSE96583', 'GSE97104', 'GSE99795',\n",
    "}\n",
    "\n",
    "new_geo_projects = [geo for geo in geo_projects if geo['geo_accession'] not in loaded_geo_accessions]\n",
    "cell_count(new_geo_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "36688"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 12
    }
   ],
   "source": [
    "loaded_shared_geo_projects = [geo for geo in geo_projects if geo['geo_accession'] in loaded_geo_accessions]\n",
    "loaded_shared_geo_accessions = [geo['geo_accession'] for geo in geo_projects]\n",
    "cell_count(loaded_shared_geo_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "659386"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 13
    }
   ],
   "source": [
    "unique_projects = [p for p in projects if p not in geo_projects and p not in hca_projects]\n",
    "cell_count(unique_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "830543"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 14
    }
   ],
   "source": [
    "non_geo_projects = [p for p in projects if p not in geo_projects]\n",
    "cell_count(non_geo_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "991902"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 15
    }
   ],
   "source": [
    "new_projects = [p for p in projects if p not in loaded_shared_geo_projects]\n",
    "cell_count(new_projects)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "already_loaded_geo_projects = {a for a in loaded_geo_accessions if a not in loaded_shared_geo_accessions}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "WARNING:root:Removing stale symlink from projects/E-CURD-10 to c062eb06-2c1d-5052-922a-5cf872155120.\n",
      "INFO:root:Linking projects/E-CURD-10 to c062eb06-2c1d-5052-922a-5cf872155120\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/experiment-metadata.zip`\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/experiment-design`\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/cluster`\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/marker-genes.zip`\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/normalised.zip`\n",
      "INFO:download_scxa:Skipping download of file `projects/c062eb06-2c1d-5052-922a-5cf872155120/scxa/quantification-raw.zip`\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "from download_scxa import download_project, list_projects\n",
    "projects = list_projects()\n",
    "accessions = [p['experimentAccession'] for p in projects]\n",
    "test_accession = accessions[0]\n",
    "download_project(test_accession, Path('projects'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "INFO:root:Expansion of projects/E-CURD-10/scxa/experiment-metadata.zip already complete\n",
      "INFO:root:Expansion of projects/E-CURD-10/scxa/normalised.zip already complete\n",
      "INFO:root:Expansion of projects/E-CURD-10/scxa/quantification-raw.zip already complete\n",
      "INFO:root:Expansion of projects/E-CURD-10/scxa/marker-genes.zip already complete\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "import extract\n",
    "extract.main()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\"projects/E-CURD-10/bundle/project_0.json\" successfully written.\n",
      "\"projects/E-CURD-10/bundle/analysis_file_0.json\" successfully written.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "from generate_metadata_scxa import generate_metadata\n",
    "test_project = Path('projects') / test_accession\n",
    "generate_metadata(test_project )\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "hca_accessions = {d['experimentAccession'] for d in hca_projects}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "INFO:/home/jesse/gi/load-project/convert_matrices.py:Projects already done: ['E-CURD-10']\n",
      "\n",
      "WARNING:/home/jesse/gi/load-project/convert_matrices.py:Projects without a project directory: ['GSE100618', 'GSE102580', 'GSE102596', 'GSE103275', 'GSE103354', 'GSE106273', 'GSE106540', 'GSE107585', 'GSE107618', 'GSE107746', 'GSE107909', 'GSE108041', 'GSE108291', 'GSE109488', 'GSE109822', 'GSE109979', 'GSE110154', 'GSE110499', 'GSE111586', 'GSE111727', 'GSE113197', 'GSE114374', 'GSE114396', 'GSE114557', 'GSE114802', 'GSE115469', 'GSE116237', 'GSE116470', 'GSE117089', 'GSE117498', 'GSE118127', 'GSE124472', 'GSE124494', 'GSE126836', 'GSE127969', 'GSE128639', 'GSE129798', 'GSE130430', 'GSE130473', 'GSE130606', 'GSE130636', 'GSE131181', 'GSE131685', 'GSE131736', 'GSE132040', 'GSE132044', 'GSE132566', 'GSE132802', 'GSE134881', 'GSE135889', 'GSE36552', 'GSE44183', 'GSE57872', 'GSE67835', 'GSE70580', 'GSE73727', 'GSE75140', 'GSE75367', 'GSE75478', 'GSE75659', 'GSE75688', 'GSE76312', 'GSE76381', 'GSE81383', 'GSE81547', 'GSE81608', 'GSE81904', 'GSE81905', 'GSE83139', 'GSE84133', 'GSE84147', 'GSE84465', 'GSE86146', 'GSE86469', 'GSE86473', 'GSE89232', 'GSE89322', 'GSE90806', 'GSE92280', 'GSE93374', 'GSE93593', 'GSE94820', 'GSE96583', 'GSE97104', 'GSE99795']\n",
      "\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "import convert_matrices\n",
    "convert_matrices.main()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}