validate/1-validation-datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create validation indication sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add DrugCentral novel indications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>status_drugcentral</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DB00389</td>\n",
       "      <td>DOID:12361</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>DB00988</td>\n",
       "      <td>DOID:14330</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   compound_id  disease_id  status_drugcentral\n",
       "5      DB00389  DOID:12361                   1\n",
       "19     DB00988  DOID:14330                   1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://github.com/dhimmel/drugcentral/blob/e80a0c966a53ce48650d98069b126801c2793517/rephetio/indications.tsv?raw=true'\n",
    "drugcentral_df = pandas.read_table(url)\n",
    "drugcentral_df = drugcentral_df[drugcentral_df.category.isnull()]\n",
    "drugcentral_df = drugcentral_df.rename(columns={'doid_id': 'disease_id', 'drugbank_id': 'compound_id'})\n",
    "drugcentral_df = drugcentral_df[['compound_id', 'disease_id']]\n",
    "drugcentral_df['status_drugcentral'] = 1\n",
    "drugcentral_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add ClinicalTrials.gov novel indications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>n_trials</th>\n",
       "      <th>status_trials</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DOID:10283</td>\n",
       "      <td>75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>DOID:11476</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  compound_id  disease_id  n_trials  status_trials\n",
       "0     DB00014  DOID:10283        75              1\n",
       "1     DB00014  DOID:11476         2              1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://github.com/dhimmel/clintrials/blob/7c65dec7b69322ca2f8ba2b170c1b3dbd92ebff8/data/DrugBank-DO-slim.tsv?raw=true'\n",
    "trial_df = pandas.read_table(url)\n",
    "trial_df = trial_df.groupby(['compound_id', 'disease_id']).apply(len).reset_index()\n",
    "trial_df.columns = 'compound_id', 'disease_id', 'n_trials'\n",
    "trial_df['status_trials'] = 1\n",
    "trial_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply to indications with an unknown category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound_id</th>\n",
       "      <th>compound_name</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>disease_name</th>\n",
       "      <th>n_trials</th>\n",
       "      <th>status_trials</th>\n",
       "      <th>status_drugcentral</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB01048</td>\n",
       "      <td>Abacavir</td>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB01048</td>\n",
       "      <td>Abacavir</td>\n",
       "      <td>DOID:9206</td>\n",
       "      <td>Barrett's esophagus</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  compound_id compound_name  disease_id         disease_name  n_trials  \\\n",
       "0     DB01048      Abacavir  DOID:10652  Alzheimer's disease         0   \n",
       "1     DB01048      Abacavir   DOID:9206  Barrett's esophagus         0   \n",
       "\n",
       "   status_trials  status_drugcentral  \n",
       "0              0                   0  \n",
       "1              0                   0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair_df = pandas.read_table('./../prediction/features/compound-disease-pairs.tsv.bz2')\n",
    "pair_df = pair_df[pair_df.category.isnull()]\n",
    "pair_df = pair_df.drop(['category', 'status'], axis='columns')\n",
    "\n",
    "pair_df = pair_df.merge(trial_df, how='left')\n",
    "pair_df = pair_df.merge(drugcentral_df, how='left')\n",
    "for column in 'status_drugcentral', 'status_trials', 'n_trials':\n",
    "    pair_df[column] = pair_df[column].fillna(0).astype(int)\n",
    "\n",
    "pair_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    202186\n",
       "1      5594\n",
       "Name: status_trials, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair_df.status_trials.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    207572\n",
       "1       208\n",
       "Name: status_drugcentral, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair_df.status_drugcentral.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pair_df.to_csv('validation-statuses.tsv', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}