Spreadsheets_Transformation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[notebook_Readme.md on github](https://github.com/KnowEnG/Spreadsheets_Transformation/blob/master/docs/notebook_Readme.md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<style>div.input {display:none;} div.output_stderr{display:none}</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "\"\"\"    To Start This Notebook Click On:    Cell > Run All    (in the jupyter menu above)    \"\"\"\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os\n",
    "import io\n",
    "import sys\n",
    "from types import SimpleNamespace\n",
    "\n",
    "import pandas as pd\n",
    "from pandas.io.common import EmptyDataError\n",
    "import numpy as np\n",
    "\n",
    "from IPython.display import display, HTML\n",
    "import ipywidgets as widgets\n",
    "import traitlets\n",
    "\n",
    "from knpackage import toolbox as kn\n",
    "\n",
    "sys.path.insert(1, '../Spreadsheets_Transformation/src')\n",
    "import spreadsheets_transformation_toolbox as stt\n",
    "from layout_notebooks import *\n",
    "\n",
    "DEFAULT_INPUT_FILES = {'transpose': 'transpose_spreadsheet.tsv', \n",
    "                       'common_samples_1': 'intersect_spreadsheet_A.tsv', \n",
    "                       'common_samples_2': 'intersect_spreadsheet_B.tsv', \n",
    "                       'merge_1': 'merge_spreadsheet_A.txt', \n",
    "                       'merge_2': 'merge_spreadsheet_B.txt',\n",
    "                       'select_rows_spreadsheet': 'select_rows_spreadsheet.tsv',\n",
    "                       'select_rows_list': 'select_rows_list.txt',\n",
    "                       'select_averages_spreadsheet': 'average_spreadsheet.tsv',\n",
    "                       'select_averages_dict': 'average_labels.tsv',\n",
    "                       'select_categorical_spreadsheet': 'select_phenotype_spreadsheet.df',\n",
    "                       'select_categorical_phenotype': 'select_phenotype_phenotype.txt',\n",
    "                       'numerical_spreadsheet': 'other_transforms_spreadsheet.tsv',\n",
    "                       'stats_spreadsheet': 'descriptive_statistic_spreadsheet.tsv',\n",
    "                      }\n",
    "\n",
    "results_dir = USER_RESULTS_DIRECTORY\n",
    "input_data_dir = USER_DATA_DIRECTORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Category to Binary')\n",
    "\n",
    "def category_column_to_binary_matrix(button):\n",
    "    \"\"\" local function - read phenotype file and selected category - write a binar samples x categories spreadsheet\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "    \n",
    "    run_parameters = {}\n",
    "    run_parameters['results_directory'] = button.results_directory\n",
    "\n",
    "    if os.path.isdir(os.path.join(input_data_dir, get_category_file_button.file_selector.value)):\n",
    "        if not get_category_file_button.file_selector.value in DIRS_INCLUDE:\n",
    "            print('Bail out')\n",
    "            return\n",
    "        \n",
    "        dir_name = os.path.join(input_data_dir, get_category_file_button.file_selector.value)\n",
    "        dir_name_full_path = os.path.join(input_data_dir, dir_name)\n",
    "        selected_files_list = [f for f in os.listdir(dir_name_full_path) if os.path.isfile(os.path.join(dir_name_full_path, f))]\n",
    "        selected_files_string = ''\n",
    "        for sel_file in selected_files_list:\n",
    "            selected_files_string += os.path.join(dir_name_full_path, sel_file + ',')\n",
    "        selected_files_string = selected_files_string[:-1]\n",
    "        \n",
    "        run_parameters['phenotype_file_name'] = selected_files_string\n",
    "        run_parameters['column_id'] = '1'\n",
    "        \n",
    "        stt.run_categorical_binary(run_parameters)\n",
    "        \n",
    "        return\n",
    "    \n",
    "    phenotype_file_name = os.path.join(input_data_dir, get_category_file_button.file_selector.value)\n",
    "    column_id = select_column_dropdown.value\n",
    "    category_df = kn.get_spreadsheet_df(phenotype_file_name).fillna('NaN')\n",
    "    cat_bin_df = stt.select_categorical_binary(category_df, column_id)\n",
    "    \n",
    "    cat_bin_result_file_name = os.path.join(button.results_directory, 'categorical_binary' + '.tsv')\n",
    "    cat_bin_df.to_csv(cat_bin_result_file_name, sep='\\t',float_format='%g')\n",
    "    \n",
    "    button.fname_list           = [cat_bin_result_file_name]\n",
    "    visualize_selected_file(button)\n",
    "    \n",
    "select_column_dropdown = widgets.Dropdown(options=[''],\n",
    "                                           value='',\n",
    "                                           description='phenotype:')\n",
    "\n",
    "def set_columns_dropdown_options(change):\n",
    "    \"\"\" set the columns options based on file selection \"\"\"\n",
    "    file_name = os.path.join(input_data_dir, change.new)\n",
    "    if os.path.isfile(file_name):\n",
    "        selected_df = pd.read_csv(file_name, sep='\\t', index_col=0, header=0)\n",
    "        columns_list = np.unique(selected_df.columns.tolist())\n",
    "        select_column_dropdown.options = columns_list\n",
    "        select_column_dropdown.value = columns_list[0]\n",
    "        \n",
    "    elif os.path.isdir(file_name):\n",
    "        selected_files_list = os.listdir(file_name)\n",
    "        if len(selected_files_list) > 0:\n",
    "            select_column_dropdown.options = ['process_directory']\n",
    "            select_column_dropdown.value = select_column_dropdown.options[0]\n",
    "    else:\n",
    "        select_column_dropdown.options = no_file_selected_columns_list\n",
    "        select_column_dropdown.value = no_file_selected_columns_list[0]\n",
    "        \n",
    "get_category_file_button                   = get_select_view_file_button_set(input_data_dir)\n",
    "\n",
    "get_category_file_button.file_selector.observe(set_columns_dropdown_options, names='value')\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_categorical_phenotype'])):\n",
    "    get_category_file_button.file_selector.value \\\n",
    "                                                = DEFAULT_INPUT_FILES['select_categorical_phenotype']\n",
    "    set_columns_dropdown_options(SimpleNamespace(new=get_category_file_button.file_selector.value))\n",
    "    \n",
    "\n",
    "cat_col_execute_button        = get_single_file_execute_button(\n",
    "                                    input_data_dir, \n",
    "                                    results_dir, \n",
    "                                    file_selector=get_category_file_button.file_selector, \n",
    "                                    button_name='Select',\n",
    "                                    )\n",
    "cat_col_execute_button.on_click(category_column_to_binary_matrix)\n",
    "\n",
    "# display control widgets:\n",
    "show_select_view_button(get_category_file_button)\n",
    "show_widget_left(select_column_dropdown)\n",
    "show_execute_button(cat_col_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Transpose')\n",
    "\n",
    "def transpose_selected_file(button):\n",
    "    \"\"\" callback for the transpose_execute_button\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with an ipywidgets.Dropdown (.file_selector) object containing \n",
    "                        the selected file name as its .value field.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "    \n",
    "    input_data_directory        = button.input_data_directory\n",
    "    input_file_name             = button.file_selector.value\n",
    "    spreadsheet_T_df            = stt.transpose_df(\n",
    "                                    kn.get_spreadsheet_df(\n",
    "                                        os.path.join(input_data_directory, input_file_name)\n",
    "                                    ))\n",
    "\n",
    "    results_directory           = button.results_directory\n",
    "    transform_name              = \"transpose\"\n",
    "    result_file_name            = stt.get_outfile_name(\n",
    "                                    results_directory, \n",
    "                                    input_file_name, \n",
    "                                    transform_name, \n",
    "                                    timestamp=False,\n",
    "                                    )\n",
    "    spreadsheet_T_df.to_csv(result_file_name, sep='\\t',float_format='%g')\n",
    "    \n",
    "    button.fname_list           = [result_file_name]\n",
    "    visualize_selected_file(button)\n",
    "    \n",
    "# get the control widgets\n",
    "get_transpose_file_button       = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['transpose'])):\n",
    "    get_transpose_file_button.file_selector.value \\\n",
    "                                = DEFAULT_INPUT_FILES['transpose']\n",
    "\n",
    "transpose_execute_button        = get_single_file_execute_button(\n",
    "                                    input_data_dir, \n",
    "                                    results_dir, \n",
    "                                    file_selector=get_transpose_file_button.file_selector, \n",
    "                                    button_name='Transpose',\n",
    "                                    )\n",
    "transpose_execute_button.on_click(transpose_selected_file)\n",
    "\n",
    "# display control widgets\n",
    "show_select_view_button(get_transpose_file_button)\n",
    "show_execute_button(transpose_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Intersect')\n",
    "\n",
    "def get_common_samples(button):\n",
    "    \"\"\" callback for common_samples_execute_button\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "\n",
    "    input_data_directory                = button.input_data_directory    \n",
    "    spreadsheet_1_file_name             = os.path.join(input_data_directory, button.file_1_selector.value)\n",
    "    spreadsheet_2_file_name             = os.path.join(input_data_directory, button.file_2_selector.value)\n",
    "    transform_name                      = \"common_samples\"\n",
    "    \n",
    "    spreadsheet_1_df                    = kn.get_spreadsheet_df(spreadsheet_1_file_name)\n",
    "    spreadsheet_2_df                    = kn.get_spreadsheet_df(spreadsheet_2_file_name)\n",
    "    \n",
    "    spreadsheet_1_df, spreadsheet_2_df  = stt.common_samples_df(spreadsheet_1_df, spreadsheet_2_df)\n",
    "    \n",
    "    results_directory                   = button.results_directory\n",
    "    result_1_file_name                  = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_1_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    spreadsheet_1_df.to_csv(result_1_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    result_2_file_name                  = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_2_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    spreadsheet_2_df.to_csv(result_2_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    button.fname_list                   = [spreadsheet_1_file_name, spreadsheet_2_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get spreadsheet 1 control widgets\n",
    "common_samples_flistbx_1_view_button    = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['common_samples_1'])):\n",
    "    common_samples_flistbx_1_view_button.file_selector.value = DEFAULT_INPUT_FILES['common_samples_1']\n",
    "\n",
    "# get spreadsheet 2 control widgets\n",
    "common_samples_flistbx_2_view_button    = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['common_samples_2'])):\n",
    "    common_samples_flistbx_2_view_button.file_selector.value = DEFAULT_INPUT_FILES['common_samples_2']\n",
    "\n",
    "# get the execute button\n",
    "file_select_dict = {    'file_1_selector': common_samples_flistbx_1_view_button.file_selector,\n",
    "                        'file_2_selector': common_samples_flistbx_2_view_button.file_selector }\n",
    "common_samples_execute_button           = get_two_files_execute_button(\n",
    "                                            input_data_dir, \n",
    "                                            results_dir, \n",
    "                                            file_select_dict=file_select_dict,\n",
    "                                            button_name='Intersect')\n",
    "common_samples_execute_button.on_click(get_common_samples)\n",
    "\n",
    "# show the controls\n",
    "show_select_view_button(common_samples_flistbx_1_view_button)\n",
    "show_select_view_button(common_samples_flistbx_2_view_button)\n",
    "show_execute_button(common_samples_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Merge')\n",
    "\n",
    "def merge_spreadsheets(button):\n",
    "    \"\"\" Callback for merge_execute_button\n",
    "        \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "\n",
    "    input_data_directory                = button.input_data_directory    \n",
    "    spreadsheet_1_file_name             = os.path.join(input_data_directory, button.file_1_selector.value)\n",
    "    spreadsheet_2_file_name             = os.path.join(input_data_directory, button.file_2_selector.value)\n",
    "    transform_name                      = \"merge\"\n",
    "    \n",
    "    spreadsheet_1_df                    = kn.get_spreadsheet_df(spreadsheet_1_file_name)\n",
    "    spreadsheet_2_df                    = kn.get_spreadsheet_df(spreadsheet_2_file_name)\n",
    "    \n",
    "    merge_df                            = stt.merge_df(spreadsheet_1_df, spreadsheet_2_df)\n",
    "    \n",
    "    results_directory                   = button.results_directory\n",
    "    merge_file_name                     = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_1_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    merge_df.to_csv(merge_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    button.fname_list                   = [merge_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get spreadsheet 1 control widgets\n",
    "merge_flistbx_1_view_button             = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['merge_1'])):\n",
    "    merge_flistbx_1_view_button.file_selector.value = DEFAULT_INPUT_FILES['merge_1']\n",
    "\n",
    "# get spreadsheet 2 control widgets\n",
    "merge_flistbx_2_view_button             = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['merge_2'])):\n",
    "    merge_flistbx_2_view_button.file_selector.value = DEFAULT_INPUT_FILES['merge_2']\n",
    "\n",
    "# get the execute button\n",
    "merge_select_dict = {    'file_1_selector': merge_flistbx_1_view_button.file_selector,\n",
    "                        'file_2_selector': merge_flistbx_2_view_button.file_selector }\n",
    "merge_execute_button                    = get_two_files_execute_button(\n",
    "                                            input_data_dir, \n",
    "                                            results_dir, \n",
    "                                            file_select_dict=merge_select_dict,\n",
    "                                            button_name='Merge')\n",
    "merge_execute_button.on_click(merge_spreadsheets)\n",
    "\n",
    "# show the controls\n",
    "show_select_view_button(merge_flistbx_1_view_button)\n",
    "show_select_view_button(merge_flistbx_2_view_button)\n",
    "show_execute_button(merge_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Select Rows')\n",
    "\n",
    "def read_a_list_file(input_file_name):\n",
    "    \"\"\" local function to read a list from a text file\n",
    "    \n",
    "    Args:\n",
    "        input_file_name:    full path name of a file containing a list\n",
    "    \n",
    "    Returns:\n",
    "        string_input:       the list that was contained in the file\n",
    "    \"\"\"\n",
    "    with open(input_file_name, 'r') as fh:\n",
    "        str_input = fh.read()\n",
    "    return list(str_input.split())\n",
    "\n",
    "def select_genes(button):\n",
    "    \"\"\" local function to read files and get common samples and write \n",
    "        using the Spreadsheets_Transformation - spreadsheets_transformation_toolbox\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "\n",
    "    input_data_directory                = button.input_data_directory    \n",
    "    spreadsheet_1_file_name             = os.path.join(input_data_directory, button.file_1_selector.value)\n",
    "    spreadsheet_2_file_name             = os.path.join(input_data_directory, button.file_2_selector.value)\n",
    "    transform_name                      = \"select_rows\"\n",
    "    \n",
    "    gene_samples_df                     = kn.get_spreadsheet_df(spreadsheet_1_file_name)\n",
    "    gene_list_df                        = read_a_list_file(spreadsheet_2_file_name)\n",
    "\n",
    "    select_spreadsheet_df               = stt.select_genes_df(gene_samples_df, gene_list_df)\n",
    "    \n",
    "    results_directory                   = button.results_directory\n",
    "    gene_rows_file_name                 = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_1_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    select_spreadsheet_df.to_csv(gene_rows_file_name, sep='\\t', float_format='%g')\n",
    "    button.fname_list                   = [gene_rows_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get spreadsheet 1 control widgets\n",
    "select_rows_flistbx_1_view_button       = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_rows_spreadsheet'])):\n",
    "    select_rows_flistbx_1_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_rows_spreadsheet']\n",
    "\n",
    "# get spreadsheet 2 control widgets\n",
    "select_rows_flistbx_2_view_button       = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_rows_list'])):\n",
    "    select_rows_flistbx_2_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_rows_list']\n",
    "\n",
    "# get the execute button\n",
    "row_file_select_dict = {    'file_1_selector': select_rows_flistbx_1_view_button.file_selector,\n",
    "                            'file_2_selector': select_rows_flistbx_2_view_button.file_selector }\n",
    "select_rows_execute_button              = get_two_files_execute_button(\n",
    "                                            input_data_dir, \n",
    "                                            results_dir, \n",
    "                                            file_select_dict=row_file_select_dict,\n",
    "                                            button_name='Select')\n",
    "select_rows_execute_button.on_click(select_genes)\n",
    "\n",
    "# show the controls\n",
    "show_select_view_button(select_rows_flistbx_1_view_button)\n",
    "show_select_view_button(select_rows_flistbx_2_view_button)\n",
    "show_execute_button(select_rows_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Average on Labels')\n",
    "\n",
    "def get_cluster_averages(button):\n",
    "    \"\"\" local function to get the averages for the dictionay selected samples in a spreadsheet\n",
    "        using the Spreadsheets_Transformation - spreadsheets_transformation_toolbox\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "\n",
    "    input_data_directory                = button.input_data_directory    \n",
    "    spreadsheet_1_file_name             = os.path.join(input_data_directory, button.file_1_selector.value)\n",
    "    spreadsheet_2_file_name             = os.path.join(input_data_directory, button.file_2_selector.value)\n",
    "    transform_name                      = \"selected_averages\"\n",
    "    \n",
    "    spreadsheet_df                      = kn.get_spreadsheet_df(spreadsheet_1_file_name)\n",
    "    labels_df                           = pd.read_csv(spreadsheet_2_file_name, \n",
    "                                                      index_col=0, \n",
    "                                                      sep='\\t', \n",
    "                                                      names=['sample','cluster_number'],\n",
    "                                                     )\n",
    "    centroid_statistic                  = button.centroid_selector.value\n",
    "    \n",
    "    averages_df                         = stt.cluster_statistics_df(spreadsheet_df, labels_df, centroid_statistic)\n",
    "    results_directory                   = button.results_directory\n",
    "    averages_file_name                  = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_1_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    averages_df.to_csv(averages_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    button.fname_list                   = [averages_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "\n",
    "# get spreadsheet 1 control widgets\n",
    "averages_flistbx_1_view_button = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_averages_spreadsheet'])):\n",
    "    averages_flistbx_1_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_averages_spreadsheet']\n",
    "\n",
    "# get spreadsheet 2 control widgets\n",
    "averages_flistbx_2_view_button    = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_averages_dict'])):\n",
    "    averages_flistbx_2_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_averages_dict']\n",
    "\n",
    "# get the execute button\n",
    "ave_file_select_dict = {    'file_1_selector': averages_flistbx_1_view_button.file_selector,\n",
    "                            'file_2_selector': averages_flistbx_2_view_button.file_selector }\n",
    "averages_execute_button           = get_two_files_execute_button(\n",
    "                                            input_data_dir, \n",
    "                                            results_dir, \n",
    "                                            file_select_dict=ave_file_select_dict,\n",
    "                                            button_name='Intersect')\n",
    "averages_execute_button.on_click(get_cluster_averages)\n",
    "\n",
    "AVAILABLE_STATISTICAL_CENTROIDS = ['std', 'median', 'mean']\n",
    "centroid_selector_dropdown = widgets.Dropdown(options=AVAILABLE_STATISTICAL_CENTROIDS,\n",
    "                                              value=AVAILABLE_STATISTICAL_CENTROIDS[0],\n",
    "                                              description='Statistic:')\n",
    "averages_execute_button.centroid_selector = centroid_selector_dropdown\n",
    "\n",
    "# show the controls\n",
    "show_select_view_button(averages_flistbx_1_view_button)\n",
    "show_select_view_button(averages_flistbx_2_view_button)\n",
    "show_widget_left(centroid_selector_dropdown)\n",
    "show_execute_button(averages_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Select Phenotype')\n",
    "\n",
    "def select_categorical(button):\n",
    "    \"\"\" local function to read files and select categorical and write \n",
    "        using the Spreadsheets_Transformation - spreadsheets_transformation_toolbox\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "    \n",
    "    input_data_directory                = button.input_data_directory    \n",
    "    spreadsheet_file_name               = os.path.join(input_data_directory, button.file_1_selector.value)\n",
    "    phenotype_file_name                 = os.path.join(input_data_directory, button.file_2_selector.value)\n",
    "    phenotype_id                        = button.phenotype_id_selector.value\n",
    "    select_category                     = button.category_selectory.value\n",
    "\n",
    "    \n",
    "    transform_name                      = \"categorical\"\n",
    "    \n",
    "    spreadsheet_df                      = kn.get_spreadsheet_df(spreadsheet_file_name)\n",
    "    phenotype_df                        = kn.get_spreadsheet_df(phenotype_file_name)\n",
    "    \n",
    "    spreadsheet_cat_df, phenotype_cat_df= stt.select_subtype_df(spreadsheet_df, \n",
    "                                                                phenotype_df, \n",
    "                                                                phenotype_id, \n",
    "                                                                select_category,\n",
    "                                                                )\n",
    "    \n",
    "    results_directory                   = button.results_directory\n",
    "    spreadsheet_cat_file_name           = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            spreadsheet_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    spreadsheet_cat_df.to_csv(spreadsheet_cat_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    phenotype_cat_file_name             = stt.get_outfile_name(\n",
    "                                            results_directory, \n",
    "                                            phenotype_file_name, \n",
    "                                            transform_name, \n",
    "                                            timestamp=False,\n",
    "                                            )\n",
    "    phenotype_cat_df.to_csv(phenotype_cat_file_name, sep='\\t', float_format='%g')\n",
    "    \n",
    "    button.fname_list                   = [spreadsheet_cat_file_name, phenotype_cat_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get spreadsheet 1 control widgets\n",
    "categorical_flistbx_1_view_button       = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_categorical_spreadsheet'])):\n",
    "    categorical_flistbx_1_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_categorical_spreadsheet']\n",
    "\n",
    "# get spreadsheet 2 control widgets\n",
    "categorical_flistbx_2_view_button       = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['select_categorical_phenotype'])):\n",
    "    categorical_flistbx_2_view_button.file_selector.value = DEFAULT_INPUT_FILES['select_categorical_phenotype']\n",
    "\n",
    "# get the execute button\n",
    "cat_file_select_dict = {'file_1_selector': categorical_flistbx_1_view_button.file_selector,\n",
    "                        'file_2_selector': categorical_flistbx_2_view_button.file_selector }\n",
    "categorical_execute_button              = get_two_files_execute_button(\n",
    "                                            input_data_dir, \n",
    "                                            results_dir, \n",
    "                                            file_select_dict=cat_file_select_dict,\n",
    "                                            button_name='Select',\n",
    "                                            )\n",
    "categorical_execute_button.on_click(select_categorical)\n",
    "        \n",
    "def reset_phenotype_id_list(change):\n",
    "    \"\"\" callback for phenotype file selector (categorical_flistbx_2_view_button.file_selector) change \"\"\"\n",
    "    new_file = os.path.join(input_data_dir, change.new)\n",
    "    new_df = pd.read_csv(new_file, sep='\\t', index_col=0, header=0)\n",
    "    new_phenotpye_list              = new_df.columns.tolist()\n",
    "    phenotype_id_listbox.options    = new_phenotpye_list\n",
    "    \n",
    "    phenotype_id_listbox.value      = new_phenotpye_list[0]\n",
    "    \n",
    "    a_list = new_df[phenotype_id_listbox.value]\n",
    "    a_list = a_list.dropna()\n",
    "    category_select_listbox.options = sorted(set(a_list))\n",
    "    category_select_listbox.value   = category_select_listbox.options[0]\n",
    "    \n",
    "phenotype_id_listbox = widgets.Dropdown(options=[''],\n",
    "                                        value='',\n",
    "                                        description='column:',\n",
    "                                       )\n",
    "categorical_flistbx_2_view_button.target_dir = input_data_dir\n",
    "categorical_flistbx_2_view_button.file_selector.observe(reset_phenotype_id_list, names='value')\n",
    "\n",
    "\n",
    "def reset_category_list(change):\n",
    "    \"\"\" callback for phenotype selection (phenotype_id_listbox) change \"\"\"\n",
    "    new_file = os.path.join(input_data_dir, categorical_flistbx_2_view_button.file_selector.value)\n",
    "    new_df = pd.read_csv(new_file, sep='\\t', index_col=0, header=0)\n",
    "    \n",
    "    if pd.isnull(phenotype_id_listbox.value):\n",
    "        return\n",
    "    a_list = new_df[phenotype_id_listbox.value]\n",
    "    a_list = a_list.dropna()\n",
    "    \n",
    "    category_select_listbox.options = sorted(set(a_list))\n",
    "    category_select_listbox.value   = category_select_listbox.options[0]\n",
    "\n",
    "category_select_listbox = widgets.Dropdown(options=[''],\n",
    "                                           value='',\n",
    "                                           description='phenotype:')\n",
    "phenotype_id_listbox.observe(reset_category_list, names='value')\n",
    "\n",
    "# initialize phenotype category and id\n",
    "categorical_flistbx_2_view_button.file_selector.current = categorical_flistbx_2_view_button.file_selector.value\n",
    "categorical_flistbx_2_view_button.file_selector.value   = categorical_flistbx_2_view_button.file_selector.options[0]\n",
    "categorical_flistbx_2_view_button.file_selector.value   = categorical_flistbx_2_view_button.file_selector.current\n",
    "reset_phenotype_id_list(SimpleNamespace(new=categorical_flistbx_2_view_button.file_selector.value))\n",
    "\n",
    "# attache the phenotype selectors to the execcute button\n",
    "categorical_execute_button.phenotype_id_selector        = phenotype_id_listbox\n",
    "categorical_execute_button.category_selectory           = category_select_listbox\n",
    "categorical_execute_button.on_click(select_categorical)\n",
    "\n",
    "# show the controls\n",
    "show_select_view_button(categorical_flistbx_1_view_button)\n",
    "show_select_view_button(categorical_flistbx_2_view_button)\n",
    "show_widget_left(widgets.VBox([phenotype_id_listbox, category_select_listbox]))\n",
    "show_execute_button(categorical_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Other Transformations')\n",
    "\n",
    "def select_numerical_transform(button):\n",
    "    \"\"\" callback for get_numerical_function_execute_button\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "    \n",
    "    input_data_directory            = button.input_data_directory\n",
    "    input_file_name                 = button.file_selector.value\n",
    "    spreadsheet_df                  = kn.get_spreadsheet_df(os.path.join(input_data_directory, input_file_name))\n",
    "\n",
    "    numeric_function                = button.numerical_function.value\n",
    "    results_directory               = button.results_directory\n",
    "    transform_name                  = numeric_function\n",
    "    \n",
    "    if transform_name == 'abs':\n",
    "        result_df                   = stt.abs_df(spreadsheet_df)\n",
    "        \n",
    "    elif transform_name == 'z_transform':\n",
    "        z_transform_axis            = button.numerical_function.z_transform_axis.value\n",
    "        z_transform_ddof            = button.numerical_function.z_transform_ddof.value\n",
    "        result_df                   = stt.z_transform_df(\n",
    "                                        spreadsheet_df, \n",
    "                                        axis=z_transform_axis, \n",
    "                                        ddof=z_transform_ddof,\n",
    "                                        )\n",
    "        \n",
    "    elif transform_name == 'log_transform':\n",
    "        log_transform_log_base      = button.numerical_function.log_transform_log_base.value\n",
    "        if log_transform_log_base == \"e\":\n",
    "            log_transform_log_base  = np.exp(1)\n",
    "        log_transform_log_offset    = button.numerical_function.log_transform_log_offset.value\n",
    "        result_df                   = stt.log_transform_df(spreadsheet_df,\n",
    "                                        log_base=log_transform_log_base,\n",
    "                                        log_offset=log_transform_log_offset,\n",
    "                                        )\n",
    "        \n",
    "    elif transform_name == 'threshold':\n",
    "        threshold_cut_off               = button.numerical_function.threshold_cut_off.value\n",
    "        threshold_substitution_value    = button.numerical_function.threshold_substitution_value.value\n",
    "        threshold_scope                 = button.numerical_function.threshold_scope.value\n",
    "        result_df                       = stt.threshold_df(spreadsheet_df,\n",
    "                                            cut_off=threshold_cut_off,\n",
    "                                            sub_val=threshold_substitution_value,\n",
    "                                            scope=threshold_scope,\n",
    "                                            )\n",
    "\n",
    "    result_file_name = stt.get_outfile_name(results_directory, input_file_name, transform_name, timestamp=False)\n",
    "    result_df.to_csv(result_file_name, sep='\\t',float_format='%g')\n",
    "    button.fname_list = [result_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get the control widgets\n",
    "get_other_transform_file_button         = get_select_view_file_button_set(input_data_dir)\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['numerical_spreadsheet'])):\n",
    "    get_other_transform_file_button.file_selector.value = DEFAULT_INPUT_FILES['numerical_spreadsheet']\n",
    "    \n",
    "\"\"\" display a numeric function options listbox and if selected is:\n",
    "    abs             - grey out all else in all cases\n",
    "    z_transform     - show radio button row or columns, and a int text box for ddof\n",
    "    log_transform   - show log_transform float base text box, and a float offset box\n",
    "    threshold       - show a float threshold cut-off, substitution box, and threshold bound [SUB_BELOW, SUB_ABOVE]\n",
    "\"\"\"\n",
    "def reset_aux_controls(change):\n",
    "    \"\"\" callback - sub controls setup for numerical_transform method options \"\"\"\n",
    "    if change.old == 'z_transform':\n",
    "        change['owner'].z_transform_axis.disabled=True\n",
    "        change['owner'].z_transform_ddof.disabled=True\n",
    "        \n",
    "    elif change.old == 'log_transform':\n",
    "        change['owner'].log_transform_log_base.disabled=True\n",
    "        change['owner'].log_transform_log_offset.disabled=True\n",
    "        \n",
    "    elif change.old == 'threshold':\n",
    "        change['owner'].threshold_cut_off.disabled=True\n",
    "        change['owner'].threshold_substitution_value.disabled=True\n",
    "        change['owner'].threshold_scope.disabled=True\n",
    "    \n",
    "    if change.new == 'abs':\n",
    "        pass\n",
    "    \n",
    "    elif change.new == 'z_transform':\n",
    "        change['owner'].z_transform_axis.disabled=False\n",
    "        change['owner'].z_transform_ddof.disabled=False\n",
    "        \n",
    "    elif change.new == 'log_transform':\n",
    "        change['owner'].log_transform_log_base.disabled=False\n",
    "        change['owner'].log_transform_log_offset.disabled=False\n",
    "\n",
    "    elif change.new == 'threshold':\n",
    "        change['owner'].threshold_cut_off.disabled=False\n",
    "        change['owner'].threshold_substitution_value.disabled=False\n",
    "        change['owner'].threshold_scope.disabled=False\n",
    "\n",
    "numerical_function_options                          = ['abs', 'z_transform', 'log_transform', 'threshold']\n",
    "numerical_function_dropdown                         = widgets.Dropdown(\n",
    "                                                        options=numerical_function_options,\n",
    "                                                        value=numerical_function_options[0],\n",
    "                                                        description='stats function')\n",
    "\n",
    "threshold_scope                                          = ['SUB_BELOW', 'SUB_ABOVE']\n",
    "log_trans_base                                           = np.exp(1)\n",
    "default_threshold                                        = 0.5\n",
    "numerical_function_dropdown.z_transform_axis             = widgets.Dropdown(\n",
    "                                                            options={'rows': 0, 'columns': 1},\n",
    "                                                            value=0, description='axis', \n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.z_transform_ddof             = widgets.IntText(\n",
    "                                                            value=0,\n",
    "                                                            description='ddof',\n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.log_transform_log_base       = widgets.FloatText(\n",
    "                                                            value=log_trans_base,\n",
    "                                                            description='log_base', \n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.log_transform_log_offset     = widgets.FloatText(\n",
    "                                                            value=0,\n",
    "                                                            description='offset', \n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.threshold_cut_off            = widgets.FloatText(\n",
    "                                                            value=default_threshold,\n",
    "                                                            description='cut-off', \n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.threshold_substitution_value = widgets.FloatText(\n",
    "                                                            value=0,\n",
    "                                                            description='substitute', \n",
    "                                                            disabled=True)\n",
    "numerical_function_dropdown.threshold_scope              = widgets.Dropdown(\n",
    "                                                            options=threshold_scope,\n",
    "                                                            value=threshold_scope[0],\n",
    "                                                            description='sub', disabled=True)\n",
    "numerical_function_dropdown.observe(reset_aux_controls, names='value')\n",
    "\n",
    "get_numerical_function_execute_button               = get_single_file_execute_button(\n",
    "                                                        input_data_dir, \n",
    "                                                        results_dir, \n",
    "                                                        file_selector=get_other_transform_file_button.file_selector, \n",
    "                                                        button_name='Calculate',\n",
    "                                                        )\n",
    "get_numerical_function_execute_button.numerical_function = numerical_function_dropdown\n",
    "get_numerical_function_execute_button.on_click(select_numerical_transform)\n",
    "\n",
    "# display control widgets\n",
    "show_select_view_button(get_other_transform_file_button)\n",
    "show_widget_left(numerical_function_dropdown)\n",
    "show_widget_left(widgets.VBox([\n",
    "                        numerical_function_dropdown.z_transform_axis,\n",
    "                        numerical_function_dropdown.z_transform_ddof,\n",
    "                        numerical_function_dropdown.log_transform_log_base,\n",
    "                        numerical_function_dropdown.log_transform_log_offset,\n",
    "                        numerical_function_dropdown.threshold_cut_off,\n",
    "                        numerical_function_dropdown.threshold_substitution_value,\n",
    "                        numerical_function_dropdown.threshold_scope,\n",
    "                        ]))\n",
    "show_execute_button(get_numerical_function_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "show_cell_title('Descriptive Statistics')\n",
    "\n",
    "def get_stats_value(button):\n",
    "    \"\"\" callback for get_stats_execute_button\n",
    "    \n",
    "    Args: \n",
    "        button:         an ipywidgets.Button object with 2 ipywidgets.Dropdown (.file_selector) objects containing \n",
    "                        the selected file names in the .value fields.\n",
    "    \"\"\"\n",
    "    if button.description == 'Clear':\n",
    "        visualize_selected_file(button)\n",
    "        return\n",
    "\n",
    "    input_data_directory                        = button.input_data_directory\n",
    "    input_file_name                             = button.file_selector.value\n",
    "    spreadsheet_df                              = kn.get_spreadsheet_df(\n",
    "                                                    os.path.join(input_data_directory, input_file_name)\n",
    "                                                    )\n",
    "    stats_function                              = button.stats_function.value\n",
    "    direction_reference                         = button.direction_reference.value\n",
    "    \n",
    "    result_df                                   = stt.stats_df(\n",
    "                                                    spreadsheet_df, \n",
    "                                                    stats_function, \n",
    "                                                    direction_reference,\n",
    "                                                    )\n",
    "    results_directory                           = button.results_directory\n",
    "    stats_function                              = button.stats_function.value\n",
    "    direction_reference                         = button.direction_reference.value\n",
    "    transform_name                              = stats_function + '_' + direction_reference\n",
    "    \n",
    "    result_file_name                            = stt.get_outfile_name(\n",
    "                                                    results_directory, \n",
    "                                                    input_file_name, \n",
    "                                                    transform_name, \n",
    "                                                    timestamp=False,\n",
    "                                                    )\n",
    "    result_df.to_csv(result_file_name, sep='\\t',float_format='%g')\n",
    "    \n",
    "    button.fname_list                           = [result_file_name]\n",
    "    visualize_selected_file(button)\n",
    "\n",
    "# get the control widgets\n",
    "get_stats_function_file_button                  = get_select_view_file_button_set(input_data_dir)\n",
    "\n",
    "if os.path.isfile(os.path.join(input_data_dir, DEFAULT_INPUT_FILES['stats_spreadsheet'])):\n",
    "    get_stats_function_file_button.file_selector.value \\\n",
    "                                                = DEFAULT_INPUT_FILES['stats_spreadsheet']\n",
    "\n",
    "stats_function_options                          = {\n",
    "                                                    'min': 'min',\n",
    "                                                    'max': 'max', \n",
    "                                                    'mean': 'mean', \n",
    "                                                    'median': 'median', \n",
    "                                                    'variance': 'variation', \n",
    "                                                    'std_deviation': 'std_deviation', \n",
    "                                                    'sum': 'sum',\n",
    "                                                    }\n",
    "\n",
    "stats_function_dropdown                         = widgets.Dropdown(\n",
    "                                                    options=stats_function_options,\n",
    "                                                    value=stats_function_options['min'],\n",
    "                                                    description='function',\n",
    "                                                    )\n",
    "\n",
    "direction_reference_options                     = ['columns', 'rows', 'all']\n",
    "direction_reference_dropdown                    = widgets.Dropdown(\n",
    "                                                    options=direction_reference_options,\n",
    "                                                    value=direction_reference_options[0],\n",
    "                                                    description='direction',\n",
    "                                                    )\n",
    "\n",
    "get_stats_execute_button                        = get_single_file_execute_button(\n",
    "                                                    input_data_dir, \n",
    "                                                    results_dir, \n",
    "                                                    file_selector=get_stats_function_file_button.file_selector, \n",
    "                                                    button_name='Calculate',\n",
    "                                                    )\n",
    "get_stats_execute_button.stats_function         = stats_function_dropdown\n",
    "get_stats_execute_button.direction_reference    = direction_reference_dropdown\n",
    "get_stats_execute_button.on_click(get_stats_value)\n",
    "\n",
    "# display control widgets\n",
    "show_select_view_button(get_stats_function_file_button)\n",
    "show_widget_left(widgets.VBox([stats_function_dropdown, direction_reference_dropdown]))\n",
    "show_execute_button(get_stats_execute_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}