SHERPA_threshold_GridSearch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "17b53023",
   "metadata": {},
   "source": [
    "# SHERPA (GridSearch) hyperparameter tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c28b945",
   "metadata": {},
   "source": [
    "## General setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "035b15d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, shutil\n",
    "from utils.setup import SetupSherpa\n",
    "from neural_networks.models import generate_input_list\n",
    "from neural_networks.models import generate_output_list\n",
    "\n",
    "argv = sys.argv[1:]\n",
    "# argv = [\"-c\", \"nn_config/# argv = [\"-c\", \"nn_config/SHERPA_threshold_GridSearch/cfg_SHERPA_GridSearch_flnt.yml\"]\n",
    "config_file = argv[-1]\n",
    "\n",
    "setup = SetupSherpa(argv)\n",
    "\n",
    "input_list        = generate_input_list(setup)\n",
    "output_list       = generate_output_list(setup)\n",
    "setup.output_list = output_list\n",
    "spcam_outputs     = setup.spcam_outputs\n",
    "children_idx_levs = setup.children_idx_levs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88dd968a",
   "metadata": {},
   "source": [
    "## SHERPA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4979bad4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sherpa\n",
    "from datetime import datetime\n",
    "from pathlib  import Path\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint\n",
    "from neural_networks.cbrain.learning_rate_schedule import LRUpdate\n",
    "from neural_networks.cbrain.utils import load_pickle\n",
    "from neural_networks.data_generator import build_train_generator,build_valid_generator\n",
    "from neural_networks.cbrain.data_generator import DataGenerator\n",
    "from neural_networks.cbrain.save_weights import save_norm\n",
    "from neural_networks.models import generate_model_sherpa\n",
    "from neural_networks.models import get_parents_sherpa"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f7fe94f",
   "metadata": {},
   "source": [
    "## Parameters & ranges\n",
    "https://parameter-sherpa.readthedocs.io/en/latest/gettingstarted/guide.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44f4822",
   "metadata": {},
   "outputs": [],
   "source": [
    "alg        = sherpa.algorithms.GridSearch()\n",
    "hp_space = {\n",
    "    'num_layers':setup.sherpa_num_layers,\n",
    "     'num_nodes':setup.sherpa_num_nodes,\n",
    "    'thresholds':setup.thresholds,\n",
    "}\n",
    "parameters = sherpa.Parameter.grid(hp_space)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2837f73",
   "metadata": {},
   "source": [
    "## Study "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "457c4c17",
   "metadata": {},
   "outputs": [],
   "source": [
    "study = sherpa.Study(parameters=parameters,\n",
    "                     algorithm=alg,\n",
    "                     lower_is_better=True,\n",
    "                     dashboard_port=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "869f1b5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.today().strftime('%Y%m%d')\n",
    "\n",
    "for j, output in enumerate(output_list):        \n",
    "\n",
    "    main_path = Path(\n",
    "        \"{nn_sherpa_path}/{date}_{output}/\".format(\n",
    "            nn_sherpa_path=setup.nn_sherpa_path,date=date,output=output\n",
    "        )\n",
    "    )\n",
    "    Path(main_path).mkdir(parents=True, exist_ok=True)\n",
    "    shutil.copyfile(config_file, Path(main_path,config_file.split('/')[-1]))\n",
    "    \n",
    "    print(f\"Output: {output}, id: {j}\")\n",
    "    setup.output        = output\n",
    "    setup.spcam_outputs = [iVar for iVar in spcam_outputs if iVar.value in output]\n",
    "    setup.children_idx_levs = [[iLev,iId] for iLev,iId in children_idx_levs \\\n",
    "                                   if str(iLev)[:3] in output]\n",
    "    inputs    = [False,input_list][setup.nn_type == 'SingleNN']\n",
    "    pc_alpha  = False\n",
    "    threshold = False\n",
    "    \n",
    "    for trial in study:\n",
    "        \n",
    "        # Hyperparameters\n",
    "        if setup.nn_type == 'CausalSingleNN':\n",
    "            setup.pc_alpha    = [setup.sherpa_pc_alphas]\n",
    "#             setup.threshold   = [setup.sherpa_thresholds]\n",
    "        setup.thresholds = [trial.parameters['thresholds']]\n",
    "        setup.num_layers = trial.parameters['num_layers']\n",
    "        setup.num_nodes  = trial.parameters['num_nodes']\n",
    "        setup.n_trial    = trial.id\n",
    "\n",
    "        # Causal links?\n",
    "        if setup.nn_type == 'CausalSingleNN':\n",
    "            inputs, pc_alpha, threshold = get_parents_sherpa(setup)\n",
    "            print(f\"\\n CausalNN with pc_alpha: {pc_alpha}; threshold: {threshold}\")\n",
    "        print(inputs)\n",
    "\n",
    "#       Create the model\n",
    "        model = generate_model_sherpa(setup,\n",
    "                                      parents=inputs,\n",
    "                                      pc_alpha=pc_alpha,\n",
    "                                      threshold=threshold)\n",
    "            \n",
    "        input_vars_dict  = model.input_vars_dict\n",
    "        output_vars_dict = model.output_vars_dict\n",
    "        \n",
    "        path = Path(str(main_path)+\"/{id_trial}/\".format(id_trial=trial.id))\n",
    "        path = model.get_path(path)\n",
    "        Path(path).mkdir(parents=True, exist_ok=True)\n",
    "        \n",
    "        print(f\"\\nTrial ({trial.id}) summary: thr-{threshold}, {setup.num_layers} layers & {setup.num_nodes} nodes\")\n",
    "        \n",
    "        with build_train_generator(\n",
    "            input_vars_dict, output_vars_dict, setup\n",
    "        ) as train_gen, build_valid_generator(\n",
    "            input_vars_dict, output_vars_dict, setup\n",
    "        ) as valid_gen:\n",
    "\n",
    "            lrs = LearningRateScheduler(\n",
    "                LRUpdate(init_lr=setup.init_lr, step=setup.step_lr, divide=setup.divide_lr)\n",
    "            )\n",
    "            early_stop = EarlyStopping(monitor=\"loss\", patience=setup.train_patience)\n",
    "            checkpoint = ModelCheckpoint(\n",
    "                str(path),\n",
    "                save_best_only=True, \n",
    "                monitor='loss',\n",
    "                mode='min'\n",
    "            )\n",
    "            callbacks  = [lrs, early_stop, checkpoint]\n",
    "            \n",
    "            # Train model\n",
    "            init_epochs = 0; epochs = setup.epochs\n",
    "            for i in range(init_epochs, epochs):\n",
    "                print(f\"initial_epoch: {i}, epochs: {i+1}\")\n",
    "                model.model.fit(train_gen, callbacks=callbacks, initial_epoch=i, epochs=i+1)\n",
    "                loss, metric = model.model.evaluate(valid_gen)\n",
    "                print(\"Validation mse: \", metric)\n",
    "                study.add_observation(trial=trial, iteration=i,\n",
    "                                      objective=metric,\n",
    "                                      context={'loss': loss})\n",
    "            \n",
    "            study.finalize(trial=trial)\n",
    "            # Save trial\n",
    "            # Save trial, weights & input list\n",
    "            filename = model.get_filename()\n",
    "            print(f\"Saving model at: \", Path(path, f\"{filename}_model.h5\"))\n",
    "            model.model.save(Path(path, f\"{filename}_model.h5\"))\n",
    "            model.model.save_weights(str(Path(path, f\"{filename}_weights.h5\")))\n",
    "            model.save_input_list(path, filename)\n",
    "            if trial.id == 1:\n",
    "                save_norm(\n",
    "                    input_transform=train_gen.input_transform,\n",
    "                    output_transform=train_gen.output_transform,\n",
    "                    save_dir=str(main_path),\n",
    "                    filename=filename,\n",
    "                )\n",
    "\n",
    "            # Save study\n",
    "            study.save(output_dir=Path(main_path))\n",
    "                \n",
    "        print()\n",
    "    print()\n",
    "\n",
    "# # Save study\n",
    "# Path(main_path).mkdir(parents=True, exist_ok=True)\n",
    "# study.save(output_dir=Path(main_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68d90962",
   "metadata": {},
   "source": [
    "## Load the study"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03c3e1d8",
   "metadata": {},
   "source": [
    "study.load_dashboard(Path(path))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2f06341",
   "metadata": {},
   "source": [
    "study.get_best_result()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "551d0e2a",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "causalnncam",
   "language": "python",
   "name": "causalnncam"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}