BioStudies NFS data access.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: jsonpath-ng in /home/awais/.local/lib/python3.8/site-packages (1.5.3)\n",
      "Requirement already satisfied: requests in /usr/lib/python3/dist-packages (2.22.0)\n",
      "Requirement already satisfied: ipyplot in /home/awais/.local/lib/python3.8/site-packages (1.1.1)\n",
      "Requirement already satisfied: tifffile in /home/awais/.local/lib/python3.8/site-packages (2022.5.4)\n",
      "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from jsonpath-ng) (1.14.0)\n",
      "Requirement already satisfied: ply in /home/awais/.local/lib/python3.8/site-packages (from jsonpath-ng) (3.11)\n",
      "Requirement already satisfied: decorator in /home/awais/.local/lib/python3.8/site-packages (from jsonpath-ng) (5.1.1)\n",
      "Requirement already satisfied: IPython in /home/awais/.local/lib/python3.8/site-packages (from ipyplot) (7.31.0)\n",
      "Requirement already satisfied: shortuuid in /home/awais/.local/lib/python3.8/site-packages (from ipyplot) (1.0.9)\n",
      "Requirement already satisfied: numpy in /home/awais/.local/lib/python3.8/site-packages (from ipyplot) (1.22.3)\n",
      "Requirement already satisfied: pillow in /usr/lib/python3/dist-packages (from ipyplot) (7.0.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (3.0.24)\n",
      "Requirement already satisfied: pexpect>4.3; sys_platform != \"win32\" in /usr/lib/python3/dist-packages (from IPython->ipyplot) (4.6.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in /usr/lib/python3/dist-packages (from IPython->ipyplot) (45.2.0)\n",
      "Requirement already satisfied: pickleshare in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (0.7.5)\n",
      "Requirement already satisfied: backcall in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (0.2.0)\n",
      "Requirement already satisfied: traitlets>=4.2 in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (5.1.1)\n",
      "Requirement already satisfied: pygments in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (2.11.2)\n",
      "Requirement already satisfied: matplotlib-inline in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (0.1.3)\n",
      "Requirement already satisfied: jedi>=0.16 in /home/awais/.local/lib/python3.8/site-packages (from IPython->ipyplot) (0.18.1)\n",
      "Requirement already satisfied: wcwidth in /home/awais/.local/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->IPython->ipyplot) (0.2.5)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /home/awais/.local/lib/python3.8/site-packages (from jedi>=0.16->IPython->ipyplot) (0.8.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install jsonpath-ng requests ipyplot tifffile --default-timeout=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import urllib.request\n",
    "import jsonpath_ng \n",
    "import ipyplot\n",
    "from jsonpath_ng import jsonpath\n",
    "from jsonpath_ng.ext import parse\n",
    "from IPython.display import display, Markdown\n",
    "from PIL import Image, ImageColor\n",
    "from numpy import asarray"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ftp_path(accession):\n",
    "    '''\n",
    "    To avoid slowing down of directory listing operations, the FTP path for each study is \n",
    "    structured as a first level directory which is generally the accession prefix without any numbers \n",
    "     and then a second level directory which is last three zero padded digits (000 if the last character \n",
    "    is not a digit, e.g., S-BIAD would be under the relative path S-BIAD/007/S-BIAD7\n",
    "    '''\n",
    "    prefix = accession.rstrip('0123456789')\n",
    "    number = accession.replace(prefix, '')\n",
    "    number = int(number)%1000 if number.isdigit() else 0    \n",
    "    return '{prefix}/{number:03d}/{accession}'.format( prefix=prefix, number=number, accession=accession)\n",
    "\n",
    "def display_attributes(attributes):\n",
    "    attribute_html = list(map( lambda node: '<tr><td><b>{}</b></td><td style=\"text-align:left\">{}</td></tr>'.format(node['name'], node['value']) ,attributes ))\n",
    "    display(Markdown('<table>{}</table>'.format(\"\".join(attribute_html))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will read study from  /mnt/c/dev/biostudies-notebooks/biostudies/S-BIAD/S-BIAD0-99/S-BIAD4/S-BIAD4.json\n"
     ]
    }
   ],
   "source": [
    "prefix = './biostudies'\n",
    "study_ftp_pattern = '{prefix}/{path}'\n",
    "study_path_pattern = '{study_root}/{accession}.json'\n",
    "\n",
    "accession = 'S-BIAD4'\n",
    "study_root = study_ftp_pattern.format( prefix = prefix, path = get_ftp_path(accession) )\n",
    "# temp replacement of first level directory -- the folloing line will be deleted after reorganisation of FTP folders\n",
    "study_root = study_root.replace('004','S-BIAD0-99')\n",
    "\n",
    "study_path = study_path_pattern.format( study_root = study_root, accession = accession)\n",
    "print ('Will read study from ', study_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<table><tr><td><b>Title</b></td><td style=\"text-align:left\">A 3D molecular atlas of the chick embryonic heart</td></tr><tr><td><b>Description</b></td><td style=\"text-align:left\">We present a detailed analysis of gene expression in the 2-day (HH12) embryonic chick heart. RNA-seq of 13 micro-dissected regions reveals regionalised expression of 15,570 genes. Of these, 132 were studied by in situ hybridisation and a subset (38 genes) was mapped by Optical Projection Tomography or serial sectioning to build a detailed 3-dimensional atlas of expression. We display this with a novel interactive 3-D viewer and as stacks of sections, revealing the boundaries of expression domains and regions of overlap. Analysis of the expression domains also defines some sub-regions distinct from those normally recognised by anatomical criteria at this stage of development, such as a previously undescribed subdivision of the atria into two orthogonal sets of domains (dorsoventral and left-right). We also include a detailed comparison of expression in the chick with the mouse and other species.</td></tr><tr><td><b>Study type</b></td><td style=\"text-align:left\">RNAseq</td></tr><tr><td><b>Study Organism</b></td><td style=\"text-align:left\">Gallus gallus</td></tr><tr><td><b>Number of Experiments</b></td><td style=\"text-align:left\">3</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Download the study json and display basic info\n",
    "with open(study_path) as file:\n",
    "    study = json.load(file)\n",
    "study_attributes = parse('$.section.attributes').find(study)[0].value\n",
    "attribute_html = list(map( lambda node: '<tr><td><b>{}</b></td><td style=\"text-align:left\">{}</td></tr>'.format(node['name'], node['value']) ,study_attributes ))\n",
    "display(Markdown('<table>{}</table>'.format(\"\".join(attribute_html))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get image List"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get image name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Histological sections of 40 markers as registered stacks of images\n"
     ]
    }
   ],
   "source": [
    "# This study has files outside the main JSON in a PageTab attribute is called `File List`\n",
    "filelists = parse('$..attributes[?(@.name == \"File List\")]').find(study)\n",
    "\n",
    "# We will display some images from the first file list here\n",
    "# Set the caption to the description of the parent section \n",
    "description_node = next( filter(lambda node: node['name']=='Description', filelists[0].context.value))\n",
    "caption = description_node['value']\n",
    "print (caption)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get Image Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching file list from /mnt/c/dev/biostudies-notebooks/biostudies/S-BIAD/S-BIAD0-99/S-BIAD4/Files/HistologicalSections-LibFile.json ...\n",
      "Got 2418 files\n"
     ]
    }
   ],
   "source": [
    "filename = filelists[0].value['value'] + '.json' \n",
    "file_endpoint = study_root + '/Files/{filename}'\n",
    "filelist_path = file_endpoint.format(accession=accession, filename = filename)\n",
    "print ('Fetching file list from {} ...'.format(filelist_path))\n",
    "with open(filelist_path, 'r') as f:\n",
    "    filelist = json.load(f)\n",
    "print ('Got {} files'.format( parse('@.`len`').find( filelist)[0].value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display first 9 images\n",
    "image_nodes = parse('@[0:9]').find(filelist)\n",
    "images = []\n",
    "labels = []\n",
    "for node in image_nodes:  \n",
    "    image_path = study_root + '/Files/' + node.value['path']    \n",
    "    images.append(asarray(Image.open(image_path)))\n",
    "    labels.append( '{} bytes'.format(node.value['size']))\n",
    "ipyplot.plot_images(images,labels,img_width=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}