tutorials/sciviewer_example_50K_Tcell.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ede7b532-5102-4de1-96d5-5b82fef16ffc",
   "metadata": {},
   "source": [
    "## This is a much bigger dataset than the 3000 PBMC sample, as it contains 50,000 T-cells. We run visualizer using sparse data as input and the performance is still quite good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "lined-conditions",
   "metadata": {},
   "outputs": [],
   "source": [
    "%gui osx\n",
    "%load_ext py5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "256bfdbf-3c0e-458f-8dd0-75dbb8cbca85",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.io import mmread\n",
    "import scipy.sparse as sparse\n",
    "from sciviewer import SCIViewer\n",
    "import scanpy as sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "assumed-truck",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b9d806dd-5030-4eca-b364-3ae18e17fc1a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DOWNLOADING AND EXTRACTING EXAMPLE DATA\n",
      "--2022-09-05 18:52:05--  https://www.dropbox.com/s/srzk17uvnhhhsgi/Tcells50k.h5ad\n",
      "Resolving www.dropbox.com (www.dropbox.com)... 162.125.4.18\n",
      "Connecting to www.dropbox.com (www.dropbox.com)|162.125.4.18|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: /s/raw/srzk17uvnhhhsgi/Tcells50k.h5ad [following]\n",
      "--2022-09-05 18:52:05--  https://www.dropbox.com/s/raw/srzk17uvnhhhsgi/Tcells50k.h5ad\n",
      "Reusing existing connection to www.dropbox.com:443.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com/cd/0/inline/BsanPLzQni5mlNCoMW0W796sutO-ioq3AJMofbTUYe6ijhWE1qqqf93VJsSLkVsbQF5BzIIAnT9902PUr3Uy__Y0q_KNhoSPjojm_lU7iyROdjkieQmvCoMk1di4aaLrJk3T6KM6qPvBL0bGs1erGWaaMvFBwMXPdEThgSkyFJN29w/file# [following]\n",
      "--2022-09-05 18:52:06--  https://uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com/cd/0/inline/BsanPLzQni5mlNCoMW0W796sutO-ioq3AJMofbTUYe6ijhWE1qqqf93VJsSLkVsbQF5BzIIAnT9902PUr3Uy__Y0q_KNhoSPjojm_lU7iyROdjkieQmvCoMk1di4aaLrJk3T6KM6qPvBL0bGs1erGWaaMvFBwMXPdEThgSkyFJN29w/file\n",
      "Resolving uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com (uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com)... 162.125.4.15\n",
      "Connecting to uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com (uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com)|162.125.4.15|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: /cd/0/inline2/BsaqK75FcNQ-o5YSvvQI5jbwmrKU4aq1VTMSNGHzHH_T1ZkV2pg9AxCVuuuCV5Nz6gIm9C5GITx2bJ3JbX-1wYfKFMRG1AOOXP4K7fAz_Wcw_o0lKbuo91ZEgIwSExqrqdCPmvlMIF_NjAy3EENYS6gAPijpOWDjViA0vcgYTsyaaw-clJYIw95bSXC_Sa5tT-LIl2j38yPEoZZ6OQ5Y0QGA-OO40jEApgwjUFq4tphVJEzFDC7H5FzNPI2B1DzwP9qs5e6SKChx8LrjFbYIFUUPG4PVICSxdRf6Hz-rMs7Dzb-Mm5__KYy9CE2Tq4pJqTdjwIMQ4tM2qJ3eLvA3lqUMi8vHhr4BAqYRZ04DIG8nk9mRy_VJQfvqarPrxz7KqImfJl98I5axkt1Lz8KqBpKgdcJqusHwEszxamqRl96EIw/file [following]\n",
      "--2022-09-05 18:52:06--  https://uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com/cd/0/inline2/BsaqK75FcNQ-o5YSvvQI5jbwmrKU4aq1VTMSNGHzHH_T1ZkV2pg9AxCVuuuCV5Nz6gIm9C5GITx2bJ3JbX-1wYfKFMRG1AOOXP4K7fAz_Wcw_o0lKbuo91ZEgIwSExqrqdCPmvlMIF_NjAy3EENYS6gAPijpOWDjViA0vcgYTsyaaw-clJYIw95bSXC_Sa5tT-LIl2j38yPEoZZ6OQ5Y0QGA-OO40jEApgwjUFq4tphVJEzFDC7H5FzNPI2B1DzwP9qs5e6SKChx8LrjFbYIFUUPG4PVICSxdRf6Hz-rMs7Dzb-Mm5__KYy9CE2Tq4pJqTdjwIMQ4tM2qJ3eLvA3lqUMi8vHhr4BAqYRZ04DIG8nk9mRy_VJQfvqarPrxz7KqImfJl98I5axkt1Lz8KqBpKgdcJqusHwEszxamqRl96EIw/file\n",
      "Reusing existing connection to uc6cdaa5f5ef9c75c533646abfdb.dl.dropboxusercontent.com:443.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 551567543 (526M) [application/octet-stream]\n",
      "Saving to: ‘../data/Tcells50k.h5ad’\n",
      "\n",
      "../data/Tcells50k.h 100%[===================>] 526.02M  8.10MB/s    in 60s     \n",
      "\n",
      "2022-09-05 18:53:06 (8.81 MB/s) - ‘../data/Tcells50k.h5ad’ saved [551567543/551567543]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('DOWNLOADING AND EXTRACTING EXAMPLE DATA')\n",
    "! mkdir -p ../data\n",
    "! wget https://www.dropbox.com/s/d41kndoe16bi1i1/Tcells50k.h5ad -O ../data/Tcells50k.h5ad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64cee3c8-c69a-4d74-8464-7d3d413d9a16",
   "metadata": {},
   "source": [
    "## Load the data as a Scanpy AnnData "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "solar-linux",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = sc.read('../data/Tcells50k.h5ad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "22b03bb8-9c85-43cd-8ad9-360e6e030845",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<52899x12563 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 28209469 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68b22aab-04c7-45a1-ab70-adbd56c308f8",
   "metadata": {},
   "source": [
    "#### the data is in sparse row format rather than sparse column so we need to convert it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4114ad71-f35c-4625-8f0a-f4907f2406c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.X = sparse.csc_matrix(data.X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc64f965-6b95-401d-9466-629cb06637ae",
   "metadata": {},
   "source": [
    "#### now you can pase the AnnData object to sciviewer and it'll use the obsm['X_umap'] attribute for the 2D embedding and the .X attribute for the expression data by default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "herbal-census",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create renderer\n",
      "Start thread\n",
      "Finish thread\n",
      "Setting up...\n",
      "0.5217690467834473 seconds to select and project cells\n",
      "Selected 6870 cells\n",
      "Calculating correlations...\n",
      "0.552480936050415 seconds to calculate correlations. Sparsity:  True\n"
     ]
    }
   ],
   "source": [
    "# gene and cell names are provided to the gene_names and cell_names attributes since\n",
    "# those don't go directly with the sparse data structure\n",
    "\n",
    "svobj = SCIViewer(data)\n",
    "svobj.explore_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "intimate-chest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>cell_name</th>\n",
       "      <th>projection</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AAACATACACCCAA-1</td>\n",
       "      <td>0.916526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>AAACATTGCTTCGC-1</td>\n",
       "      <td>0.051023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>AAACCGTGTCACCC-1</td>\n",
       "      <td>0.862504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18</td>\n",
       "      <td>AAACGCACGGTACT-1</td>\n",
       "      <td>0.116701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26</td>\n",
       "      <td>AAACGGCTGCAGAG-1</td>\n",
       "      <td>0.516208</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index         cell_name  projection\n",
       "0      0  AAACATACACCCAA-1    0.916526\n",
       "1      7  AAACATTGCTTCGC-1    0.051023\n",
       "2     15  AAACCGTGTCACCC-1    0.862504\n",
       "3     18  AAACGCACGGTACT-1    0.116701\n",
       "4     26  AAACGGCTGCAGAG-1    0.516208"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## This attribute get updated in real time whenever cells are selected\n",
    "svobj.selected_cells.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b4d3d6af-d956-48f0-9d83-e937387a813b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>R</th>\n",
       "      <th>P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DUSP2</th>\n",
       "      <td>0.449746</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GZMH</th>\n",
       "      <td>-0.787988</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPL13</th>\n",
       "      <td>0.595750</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPS2</th>\n",
       "      <td>0.474982</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IL7R</th>\n",
       "      <td>0.444177</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              R    P\n",
       "DUSP2  0.449746  0.0\n",
       "GZMH  -0.787988  0.0\n",
       "RPL13  0.595750  0.0\n",
       "RPS2   0.474982  0.0\n",
       "IL7R   0.444177  0.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8255929946899414 seconds to select and project cells\n",
      "Selected 13160 cells\n",
      "Calculating differential expression...\n",
      "0.00922703742980957 seconds to calculate genesums. Sparsity:  True\n",
      "0.07808995246887207 seconds to calculate squared genesums. Sparsity:  True\n",
      "0.7021546363830566 seconds to calculate differential expression. Sparsity:  True\n",
      "Selected gene RPL11\n",
      "Min/max expression level for gene RPL11 0.0 5.7148066\n"
     ]
    }
   ],
   "source": [
    "## This gets updated in real time when cells are selected in directional mode\n",
    "svobj.results_proj_correlation.sort_values(by='P').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "096dca12-b1c8-4aba-a51a-7ee417469f55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>T</th>\n",
       "      <th>P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PMAIP1</th>\n",
       "      <td>-15.761498</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SLC1A7</th>\n",
       "      <td>-16.943316</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPLP0</th>\n",
       "      <td>15.777176</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HLA-C</th>\n",
       "      <td>-31.591606</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HLA-B</th>\n",
       "      <td>-37.330349</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                T    P\n",
       "PMAIP1 -15.761498  0.0\n",
       "SLC1A7 -16.943316  0.0\n",
       "RPLP0   15.777176  0.0\n",
       "HLA-C  -31.591606  0.0\n",
       "HLA-B  -37.330349  0.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## This gets updated in real time when cells are selected in differential mode\n",
    "svobj.results_diffexpr.sort_values(by='P').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aa9a818-ad52-4026-b551-3ece678998bc",
   "metadata": {},
   "source": [
    "## You can re-run the analysis using dense data and see how much slower it is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dated-bargain",
   "metadata": {},
   "outputs": [],
   "source": [
    "expr_dense = pd.DataFrame(data.X.todense(), columns=data.var.index, index=data.obs.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "conscious-liberal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LINC00115</th>\n",
       "      <th>FAM41C</th>\n",
       "      <th>NOC2L</th>\n",
       "      <th>KLHL17</th>\n",
       "      <th>PLEKHN1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAACATACACCCAA-1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACATACCCCTCA-1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACATACCGGAGA-1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACATACTCTTCA-1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACATACTGGATC-1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  LINC00115  FAM41C  NOC2L  KLHL17  PLEKHN1\n",
       "AAACATACACCCAA-1        0.0     0.0    0.0     0.0      0.0\n",
       "AAACATACCCCTCA-1        0.0     0.0    0.0     0.0      0.0\n",
       "AAACATACCGGAGA-1        0.0     0.0    0.0     0.0      0.0\n",
       "AAACATACTCTTCA-1        0.0     0.0    0.0     0.0      0.0\n",
       "AAACATACTGGATC-1        0.0     0.0    0.0     0.0      0.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expr_dense.iloc[:5, :5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "lightweight-arnold",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EXPORTING DATA...\n",
      "BYE\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-05 18:54:20.343 python[55569:1345364] NewtNSView::dealloc: softLock still hold @ dealloc!\n"
     ]
    }
   ],
   "source": [
    "umap = data.obsm['X_umap']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67c9faf0-7778-4454-bd5e-fd89d0c515f7",
   "metadata": {},
   "source": [
    "#### Here we pass the expression data as a Pandas DataFrame and the umap as a Numpy ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "driven-turner",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create renderer\n",
      "Start thread\n",
      "Finish thread\n",
      "Setting up...\n",
      "0.5447080135345459 seconds to select and project cells\n",
      "Selected 7000 cells\n",
      "Calculating correlations...\n",
      "2.945772171020508 seconds to calculate correlations. Sparsity:  False\n",
      "EXPORTING DATA...\n",
      "BYE\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-05 18:54:56.581 python[55569:1345364] NewtNSView::dealloc: softLock still hold @ dealloc!\n"
     ]
    }
   ],
   "source": [
    "svobj_dense = SCIViewer(expr_dense, umap)\n",
    "svobj_dense.explore_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01aa1eb4-835a-4c04-9893-4d9d79bba9ae",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}