Trophic_model_for_gut_data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trophic model for gut data processing\n",
    "This file is used to pre-process all data (especially Chia network and Thai Children data) into the format which is convenient for simulations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "########### Self-customized setting\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   metabolites_ID  microbes_ID  \\\n",
      "0            2001          896   \n",
      "1            2001          832   \n",
      "2            2001          831   \n",
      "3            2001          600   \n",
      "4            2001          571   \n",
      "\n",
      "   edge_types (2 represents intake, 3 represents secretion and 5 represents intake and secretion)  \n",
      "0                                                  2                                               \n",
      "1                                                  2                                               \n",
      "2                                                  2                                               \n",
      "3                                                  3                                               \n",
      "4                                                  3                                               \n",
      "###################################################################################################\n",
      "                          Names\n",
      "IDs                            \n",
      "3    Acetivibrio cellulolyticus\n",
      "4      Acetobacter pasteurianus\n",
      "5      Acetohalobium arabaticum\n",
      "6        Acholeplasma laidlawii\n",
      "7      Achromobacter piechaudii\n",
      "###################################################################################################\n",
      "[2044 2060 2061 2064 2065 2068 2069 2072 2074 2079 2082 2094 2098 2149\n",
      " 2161 2172 2174 2242 2157 2206]\n",
      "###################################################################################################\n",
      "   BK301  BK303  BK304  BK305  BK306  BK307  BK308  BK309  BK310  BK311  ...  \\\n",
      "0      0      0      0      0      0      0      0      0      0      0  ...   \n",
      "1      0      0      0      0      4      0      0      0      0      0  ...   \n",
      "2      6      4     14      0     18     61     24     86      3     30  ...   \n",
      "3      6      4      9      0      0      0      0     22      1      3  ...   \n",
      "4      0      0      0      0      0      0      1      0      0      0  ...   \n",
      "\n",
      "   BR320  BR321  BR322  BR323  BR324  BR325  BR326  BR327  BR328  BR329  \n",
      "0      0      0      0      0      0      0      1      0      0      0  \n",
      "1      0      0      0      0      0      0      1      0      0      0  \n",
      "2      0      0      5      0      9      0      0     58     12      6  \n",
      "3      1      1      0      3      0     21     10     28     10      3  \n",
      "4      0      0      0      0      0      0      0      0      0      0  \n",
      "\n",
      "[5 rows x 45 columns]\n",
      "###################################################################################################\n",
      "    BK301  BK303   BK304  BK305   BK306   BK307    BK308   BK309    BK310  \\\n",
      "0  2746.0   23.0  1019.0  780.0   313.0   377.0    292.0  1058.0   1094.0   \n",
      "1  3196.0  205.0  1543.0    0.0   688.0  1369.0      0.0   454.0    827.0   \n",
      "2     0.0   15.0     0.0    0.0    16.0    20.0      0.0    24.0      0.0   \n",
      "3   153.0   43.0    96.0  154.0  3433.0   370.0   5987.0   112.0   5122.0   \n",
      "4   662.0  555.0   347.0  454.0  1335.0  9326.0  18560.0   255.0  30338.0   \n",
      "\n",
      "    BK311  ...   BR318   BR319   BR320   BR321   BR322   BR323   BR325  \\\n",
      "0   125.0  ...    74.0   178.0    44.0    46.0    36.0   227.0    42.0   \n",
      "1  4834.0  ...  1753.0   854.0  1123.0  3711.0  2013.0   270.0  3588.0   \n",
      "2     0.0  ...    14.0     0.0     0.0    17.0     0.0     0.0     0.0   \n",
      "3    86.0  ...   449.0   222.0    69.0    91.0  1524.0    60.0   308.0   \n",
      "4  2067.0  ...   728.0  5806.0   105.0   707.0   430.0  1221.0   292.0   \n",
      "\n",
      "    BR327   BR328    BR329  \n",
      "0    68.0    93.0   2108.0  \n",
      "1   898.0  1415.0      0.0  \n",
      "2     0.0     0.0      0.0  \n",
      "3    52.0   298.0  11252.0  \n",
      "4  1131.0  1599.0  75884.0  \n",
      "\n",
      "[5 rows x 42 columns]\n",
      "###################################################################################################\n",
      "Intersection between metagenome and metabolome:\n",
      "   BK301  BK303  BK304  BK305  BK306  BK307  BK308  BK309  BK310  BK311  ...  \\\n",
      "0      0      0      0      0      0      0      0      0      0      0  ...   \n",
      "1      0      0      0      0      4      0      0      0      0      0  ...   \n",
      "2      6      4     14      0     18     61     24     86      3     30  ...   \n",
      "3      6      4      9      0      0      0      0     22      1      3  ...   \n",
      "4      0      0      0      0      0      0      1      0      0      0  ...   \n",
      "\n",
      "   BR318  BR319  BR320  BR321  BR322  BR323  BR325  BR327  BR328  BR329  \n",
      "0      0      0      0      0      0      0      0      0      0      0  \n",
      "1      5      0      0      0      0      0      0      0      0      0  \n",
      "2     98      2      0      0      5      0      0     58     12      6  \n",
      "3     15      1      1      1      0      3     21     28     10      3  \n",
      "4      0      0      0      0      0      0      0      0      0      0  \n",
      "\n",
      "[5 rows x 41 columns]\n",
      "    BK301  BK303   BK304  BK305   BK306   BK307    BK308   BK309    BK310  \\\n",
      "0  2746.0   23.0  1019.0  780.0   313.0   377.0    292.0  1058.0   1094.0   \n",
      "1  3196.0  205.0  1543.0    0.0   688.0  1369.0      0.0   454.0    827.0   \n",
      "2     0.0   15.0     0.0    0.0    16.0    20.0      0.0    24.0      0.0   \n",
      "3   153.0   43.0    96.0  154.0  3433.0   370.0   5987.0   112.0   5122.0   \n",
      "4   662.0  555.0   347.0  454.0  1335.0  9326.0  18560.0   255.0  30338.0   \n",
      "\n",
      "    BK311  ...   BR318   BR319   BR320   BR321   BR322   BR323   BR325  \\\n",
      "0   125.0  ...    74.0   178.0    44.0    46.0    36.0   227.0    42.0   \n",
      "1  4834.0  ...  1753.0   854.0  1123.0  3711.0  2013.0   270.0  3588.0   \n",
      "2     0.0  ...    14.0     0.0     0.0    17.0     0.0     0.0     0.0   \n",
      "3    86.0  ...   449.0   222.0    69.0    91.0  1524.0    60.0   308.0   \n",
      "4  2067.0  ...   728.0  5806.0   105.0   707.0   430.0  1221.0   292.0   \n",
      "\n",
      "    BR327   BR328    BR329  \n",
      "0    68.0    93.0   2108.0  \n",
      "1   898.0  1415.0      0.0  \n",
      "2     0.0     0.0      0.0  \n",
      "3    52.0   298.0  11252.0  \n",
      "4  1131.0  1599.0  75884.0  \n",
      "\n",
      "[5 rows x 41 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tongwang/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:10: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/Users/tongwang/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:16: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "########### Load Chia network as net (containing information of metabolite consumption and production)\n",
    "net = pd.read_csv('./data/pruned_chia_network.csv')\n",
    "mean_net = net.groupby('microbes_ID').mean()\n",
    "selfish_net = mean_net[mean_net.iloc[:,1] == 2]\n",
    "i_selfish = selfish_net.index.values   #### i_selfish returns IDs of microbes don't generate byproducts\n",
    "print(net.head())\n",
    "print('###################################################################################################')\n",
    "\n",
    "########### Load names of all nodes in the Chia network\n",
    "names = pd.read_csv('./data/names_ID.txt',sep=': ')\n",
    "names.set_index('IDs', inplace=True)\n",
    "print(names.head())\n",
    "print('###################################################################################################')\n",
    "\n",
    "########### Load names of all nodes in the Chia network\n",
    "i_intake = pd.read_csv('./data/nutrient_intake_ID.txt',sep=': ')\n",
    "i_intake = i_intake['IDs'].values\n",
    "print(i_intake)\n",
    "print('###################################################################################################')\n",
    "\n",
    "########### Load all gut metagenomic data of all 41 individuals\n",
    "metagenome_all = pd.read_csv('./data/abundance_matched_thai.txt', sep='\\t')\n",
    "metagenome_all.head()\n",
    "metagenome_all = metagenome_all.groupby('Chia_id').sum().iloc[1:,].reset_index()\n",
    "metagenome_ID = metagenome_all['Chia_id']\n",
    "#print((metagenome_ID!=0).sum())\n",
    "metagenome = metagenome_all[metagenome_ID!=0].iloc[:,3:]\n",
    "metagenome_ID = metagenome_ID[metagenome_ID!=0]\n",
    "print(metagenome.head())\n",
    "print('###################################################################################################')\n",
    "\n",
    "########### Load all gut metabolome data of all 41 individuals\n",
    "metabolome_all = pd.read_excel('./data/metabolome_matched_thai_modified_by_Tong.xlsx')\n",
    "metabolome_all = metabolome_all.groupby('Chia_id').sum().iloc[1:,].reset_index()\n",
    "metabolome_ID = metabolome_all['Chia_id']\n",
    "#print((metabolome_ID!=0).sum())\n",
    "metabolome = metabolome_all[metabolome_ID!=0].iloc[:,2:]\n",
    "metabolome_ID = metabolome_ID[metabolome_ID!=0]\n",
    "print(metabolome.head())\n",
    "print('###################################################################################################')\n",
    "\n",
    "intersected_names = np.intersect1d(metagenome.columns.values, metabolome.columns.values)\n",
    "metagenome = metagenome[intersected_names]\n",
    "metabolome = metabolome[intersected_names]\n",
    "print('Intersection between metagenome and metabolome:')\n",
    "print(metagenome.head())\n",
    "print(metabolome.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7766192895582811"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metagenome_all = pd.read_csv('./data/abundance_matched_thai.txt', sep='\\t')\n",
    "i_match = metagenome_all['Match']=='T'\n",
    "coverage_list = []\n",
    "for col in metagenome.columns:\n",
    "    coverage_list = coverage_list + [np.sum(metagenome_all[col][i_match])  / np.sum(metagenome_all[col])]\n",
    "np.mean(coverage_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "########### pickle all processed data which are useful for simulations\n",
    "import pickle\n",
    "\n",
    "pickle_out = open(\"Chia_network.pickle\",\"wb\")\n",
    "#pickle.dump([net, i_selfish, i_intake, names], pickle_out)\n",
    "pickle.dump([net, i_selfish, i_intake, names], pickle_out, protocol=2)\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data.pickle\",\"wb\")\n",
    "#pickle.dump([metagenome_ID, metagenome, metabolome_ID, metabolome], pickle_out)\n",
    "pickle.dump([metagenome_ID, metagenome, metabolome_ID, metabolome], pickle_out, protocol=2)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "nteract": {
   "version": "0.21.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}