2_estimate_DSC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import container\n",
    "from matplotlib.ticker import ScalarFormatter\n",
    "import matplotlib\n",
    "from models.model_utils import process_image\n",
    "from utils.notebook_utils import apply_temperature, pixel_uncertainty, convolve_uncertainty\n",
    "from utils.notebook_utils import softmax, estimate_dice_li, get_patches, dist2center, scale_temp_dice\n",
    "from models.model_utils import dice_metric\n",
    "import cv2\n",
    "import os\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.patches as patches\n",
    "from glob import glob\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from munch import munchify\n",
    "import yaml\n",
    "import warnings\n",
    "import copy\n",
    "# plt.rcParams['text.usetex'] = True"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Configure split manually and select configuration of no.patches and patch size (config suffix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split = 'test'\n",
    "config_suffix = 'all_patches'\n",
    "\n",
    "split_suffix = 'val' if split == 'val' else '' # enables consistent naming convention\n",
    "run_again = False # whether to re-compute results or load from pre-computed files\n",
    "run_again_patch_selection = False # separate flag for patch selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Experiments Configs and Paths\n",
    "with open('config_' + config_suffix + '.yaml', 'r') as file0:\n",
    "        cfg = munchify(yaml.load(file0, Loader=yaml.FullLoader))\n",
    "        file0.close()\n",
    "\n",
    "with open('paths.yaml', 'r') as file0:\n",
    "        cfg_paths = munchify(yaml.load(file0, Loader=yaml.FullLoader))\n",
    "        file0.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unit Test\n",
    "if cfg.patch_size % 2  == 0:\n",
    "    warnings.warn('Please specify an uneven patchsize in config_' + config_suffix + '.yaml.')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load forward passes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Ensemble Passes for Test Set\n",
    "passes = 'ensemble_dict_val.pkl' if split == 'val' else 'ensemble_dict.pkl'\n",
    "with open('cache/' + passes, 'rb') as file2:\n",
    "    ensemble_dict = pkl.load(file2)\n",
    "    file2.close()\n",
    "\n",
    "with open('cache/' +  config_suffix + '_optimal_T.pkl', 'rb') as file3:\n",
    "    T_remain = pkl.load(file3)\n",
    "    file3.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute means over ensemble outputs and Calibrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble_dict['mean logits'] = np.array(ensemble_dict['logits']).mean(axis=0)\n",
    "ensemble_dict['mean predictions'] = softmax(np.array(ensemble_dict['logits'])).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_logits_arr = np.array(ensemble_dict['logits'])\n",
    "predictions_arr = softmax(predicted_logits_arr) # not calibrated yet -- happens later\n",
    "\n",
    "mean_true_dice = np.array(ensemble_dict['true dice']).mean(axis=0)\n",
    "pos_pred = (ensemble_dict['mean predictions'] > 0.5).mean(axis=(1,2))\n",
    "\n",
    "# deprecated:\n",
    "# predicted_logits_arr = np.log(predictions_arr / (1 - predictions_arr + 10e-9))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Uncertainty Maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_maps = []\n",
    "\n",
    "for i in range(predictions_arr.shape[1]):\n",
    "    entropy_maps.append(pixel_uncertainty(predictions_arr[:, i,: ,:], 'entropy'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Potentially Subset to Analyze Subgroups (To reproduce paper select entire test set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select relevant images\n",
    "sufficient_foreground = pos_pred > cfg.fg_thresh\n",
    "sufficient_dice = mean_true_dice > cfg.dice_thresh\n",
    "\n",
    "image_selection = np.logical_and(sufficient_foreground, sufficient_dice)\n",
    "image_selection_ids = np.where(image_selection)[0]\n",
    "print('You have selected ' + str(image_selection.sum()) + ' images.')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Patches and Estimate Remaining Dice"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select Patches with Highest Summed Uncertainty, Mask them and Estimate Dice of the Remaining Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reconstruct train and val split\n",
    "root = cfg_paths.FIVES\n",
    "train_x = sorted(glob(os.path.join(root, 'train/Original/*')))\n",
    "train_y = sorted(glob(os.path.join(root, 'train/Ground truth/*')))\n",
    "\n",
    "validation_split = .2\n",
    "indices = list(range(len(train_x)))\n",
    "split_val = int(np.floor(validation_split * len(train_x)))\n",
    "np.random.seed(23) # manually confirmed with the training script\n",
    "np.random.shuffle(indices)\n",
    "\n",
    "train_indices, val_indices = indices[split_val:], indices[:split_val]\n",
    "\n",
    "val_x = train_x[:split_val]\n",
    "val_y = train_y[:split_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load filenames\n",
    "if split == 'val':\n",
    "    split_x = val_x\n",
    "    split_y = val_y\n",
    "    print('Indexed Images from Validation Split.')\n",
    "\n",
    "elif split == 'test':\n",
    "    split_x = sorted(glob(os.path.join(root, split + '/Original/*')))\n",
    "    split_y = sorted(glob(os.path.join(root, split + '/Ground truth/*')))\n",
    "    print('Indexed images from test split.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_estimation_info(img_idx):\n",
    "    # Image and GT\n",
    "    x_path = split_x[img_idx]\n",
    "    y_path = split_y[img_idx]\n",
    "\n",
    "    image = cv2.imread(x_path, cv2.IMREAD_COLOR) ## (512, 512, 3)\n",
    "    # image = clahe_equalized(image)\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    image = cv2.resize(image, (512,512))\n",
    "\n",
    "    mask = cv2.imread(y_path, cv2.IMREAD_GRAYSCALE)  ## (512, 512)\n",
    "    mask = cv2.resize(mask, (512,512)) # interpolation =  INTER_NEAREST\n",
    "\n",
    "    x_image, y_image = process_image(image, mask)\n",
    "    gt = y_image.squeeze()\n",
    "\n",
    "    if cfg.uncertainty_type == 'entropy':\n",
    "        uncertainty_map = - entropy_maps[img_idx]\n",
    "    # elif cfg.uncertainty_type == 'variance':\n",
    "    #    uncertainty_map = variance_maps[img_idx]\n",
    "\n",
    "    convolution_dict = convolve_uncertainty(uncertainty_map, cfg.patch_size - 1)\n",
    "    convolve_map = np.array(convolution_dict['patch uncertainty']).reshape(432, 432)\n",
    "\n",
    "    row, col = np.unravel_index(np.argmax(convolve_map, axis=None), convolve_map.shape)\n",
    "    \n",
    "    ref_array = np.array(convolution_dict['reference'])\n",
    "\n",
    "    # available to choose from\n",
    "    in_image = np.ones_like(convolve_map.flatten()).astype(bool)\n",
    "    patch_references = []\n",
    "\n",
    "    for i in range(cfg.no_patches - 1):\n",
    "        # choose pixel with largest value available\n",
    "        max_idx_flat = convolve_map.flatten()[in_image].argmax()\n",
    "        # store reference point (in fullsize image space) for the patch\n",
    "        reference_point = np.array(convolution_dict['reference'])[in_image.flatten(), :][max_idx_flat, :]\n",
    "        patch_references.append(reference_point)\n",
    "\n",
    "        # update in_image, i.e. \"remove\" all pixels that are already selected by earlier patches\n",
    "        y_constraint = np.logical_and(ref_array[:, 0] <= reference_point[0] + cfg.patch_size, ref_array[:, 0] >= reference_point[0] - cfg.patch_size)\n",
    "        x_constraint = np.logical_and(ref_array[:, 1] <= reference_point[1] + cfg.patch_size, ref_array[:, 1] >= reference_point[1] - cfg.patch_size)\n",
    "        patch_constraint = np.logical_and(y_constraint, x_constraint)\n",
    "\n",
    "        in_image[patch_constraint] = False\n",
    "\n",
    "\n",
    "    prediction = predictions_arr[0, img_idx,:,:] > 0.5\n",
    "    masked_out = np.zeros_like(prediction).astype(bool)\n",
    "    dsc_curve = []\n",
    "    est_dsc_curve = []\n",
    "    est_w = []\n",
    "    true_w = []\n",
    "    est_dice_remain_list = []\n",
    "    dice_remain_list = []\n",
    "    pos_remain = []\n",
    "    masks =[]\n",
    "\n",
    "    for i in range(len(patch_references) + 1):\n",
    "\n",
    "        masks.append(~masked_out)\n",
    "\n",
    "        dice_remain = dice_metric(gt[~masked_out], prediction[~masked_out])\n",
    "        est_dice_remain = estimate_dice_li(predictions_arr[0, img_idx,:,:][~masked_out])\n",
    "\n",
    "        est_dice_remain_list.append(est_dice_remain)\n",
    "        dice_remain_list.append(dice_remain)\n",
    "        pos_remain.append((predictions_arr[0, img_idx, :,:][~masked_out] > 0.5).mean())\n",
    "\n",
    "        # update mask\n",
    "        if i <= len(patch_references) - 1:\n",
    "            patch_ref = patch_references[i]\n",
    "            masked_out[patch_ref[0]:patch_ref[0] + cfg.patch_size, patch_ref[1]:patch_ref[1] + cfg.patch_size] = True\n",
    "\n",
    "\n",
    "    return {'true dice': dice_remain_list, 'est dice': est_dice_remain_list,\n",
    "            'positive remaining': pos_remain, 'masks': masks}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if run_again_patch_selection:\n",
    "    \n",
    "    # Re-run patch selection if necessary \n",
    "    # Requires ~90min for 200imgs on one consumer CPU\n",
    "\n",
    "    estimation_info = []\n",
    "\n",
    "    for img_id in image_selection_ids:\n",
    "        estimation_info.append(get_estimation_info(img_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if run_again_patch_selection:\n",
    "    with open('cache/' + config_suffix + '_estimation_info_' + split_suffix + '.pkl', 'wb') as file:\n",
    "            pkl.dump(estimation_info, file)\n",
    "            file.close()\n",
    "else:\n",
    "    with open('cache/' + config_suffix + '_estimation_info' + split_suffix + '.pkl', 'rb') as file:\n",
    "        estimation_info = pkl.load(file)\n",
    "        file.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`estimation_info` contains a list of no.images (200 if entire test set selected).\n",
    "Each element contains a dict with lists true, est. DSC (uncalibrated!), fraction of pixels that were classified as positive and the respective binary masks.\n",
    "Each element of those refers to one patch, where entry 0 has no patches selected."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calibrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "relevant_logits = predicted_logits_arr.mean(axis=0)[image_selection_ids, :, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_remain = [] \n",
    "\n",
    "for i, one_img_logit in enumerate(relevant_logits):\n",
    "\n",
    "    for mask in estimation_info[i]['masks']:\n",
    "\n",
    "        logit_remain.append(one_img_logit[mask])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_dice_flat = [el for sublist in estimation_info for el in sublist['true dice']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_temperature_remain(T, logits):\n",
    "\n",
    "    \"\"\"\n",
    "        logits_cali: [list]\n",
    "        dice_true: [list]\n",
    "        T: [float]\n",
    "    \"\"\"\n",
    "\n",
    "    n_imgs = len(logits)\n",
    "    ts_logits = [logits_img / T for logits_img in logits]\n",
    "    ts_pyIx = [softmax(logits) for logits in ts_logits]\n",
    "\n",
    "    est_dice_cali = [estimate_dice_li(probs) for probs in ts_pyIx]\n",
    "\n",
    "\n",
    "\n",
    "    temperature_dict = {'estimated dice': est_dice_cali, 'predictions': ts_pyIx,\n",
    "                        'T': T, }\n",
    "\n",
    "    return temperature_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rescaled_probabilities = apply_temperature_remain(T_remain, logit_remain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del logit_remain # free RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_scaled_estimate_remain = rescaled_probabilities['estimated dice']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimates_matrix_ts = np.array(ts_scaled_estimate_remain).reshape(-1, cfg.no_patches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if run_again:\n",
    "    with open('cache/' + config_suffix + '_estimates_matrix.pkl', 'wb') as file:\n",
    "        pkl.dump(estimates_matrix_ts, file)\n",
    "        file.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From now on we operate with `estimates_matrix_ts` and `ts_scaled_estimate_remain`, which contain the estimated DSC for the remaining parts of the images (after the patches have been cut out)."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Impute Oracle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convolve_uncertainty(uncertainty_map, patch_size, stride=1):\n",
    "    \"\"\"\n",
    "        ::param::    \n",
    "        uncertainty_map: [np.array] of shape nxn\n",
    "        patch_size: [int] defines side length of square patch\n",
    "        stride: [int]\n",
    "        \n",
    "        ::return::\n",
    "        Patch Identifier with Convolution Value\n",
    "    \"\"\"\n",
    "\n",
    "    patch_convolved = []\n",
    "    reference_points = [] # identifies patches\n",
    "    patch_dict = {'reference': reference_points,\n",
    "                  'patch uncertainty': patch_convolved}\n",
    "    \n",
    "\n",
    "    def sliding_window(arr, step_size=stride, window_size=patch_size):\n",
    "        \"\"\"\n",
    "            Iterator which yields a binary mask for patch extraction\n",
    "            alongside identifying coordinates of the reference point.\n",
    "        \"\"\"\n",
    "        for y in range(0, arr.shape[0] - patch_size, step_size):\n",
    "            for x in range(0, arr.shape[1] - patch_size, step_size):\n",
    "\n",
    "                bool_img = np.zeros_like(arr) * False\n",
    "                bool_img[y:y + window_size, x:x + window_size] = True\n",
    "                \n",
    "                yield (y, x, bool_img.astype(bool))\n",
    "\n",
    "    windows = sliding_window(uncertainty_map, stride, patch_size)\n",
    "    \n",
    "\n",
    "    for y, x, window in windows:\n",
    "        patch_value = uncertainty_map[window].sum()\n",
    "        patch_convolved.append(patch_value)\n",
    "        reference_points.append((y, x))\n",
    "\n",
    "    return patch_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convolve a random image to obtain the reference points, required in the\n",
    "# following function\n",
    "convolution_dict = convolve_uncertainty(np.ones(shape=(512, 512)), cfg.patch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def oracle_and_random(img_idx, selection_idx):\n",
    "\n",
    "    # oracle\n",
    "\n",
    "    dsc_curve = []\n",
    "    est_dsc_curve = []\n",
    "    est_w = []\n",
    "    true_w = []\n",
    "    est_dice_remain_list = []\n",
    "    dice_remain_list = []\n",
    "    pos_remain = []\n",
    "\n",
    "    masks = estimation_info[selection_idx]['masks']\n",
    "\n",
    "    # Retrieve Prediction and Ground Truth\n",
    "    prediction = ensemble_dict['mean predictions'][img_idx, :,:]\n",
    "    ## Ground Truth\n",
    "    x_path = split_x[img_idx]\n",
    "    y_path = split_y[img_idx]\n",
    "\n",
    "    image = cv2.imread(x_path, cv2.IMREAD_COLOR) ## (512, 512, 3)\n",
    "    # image = clahe_equalized(image)\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    image = cv2.resize(image, (512,512))\n",
    "\n",
    "    mask = cv2.imread(y_path, cv2.IMREAD_GRAYSCALE)  ## (512, 512)\n",
    "    mask = cv2.resize(mask, (512,512)) # interpolation =  INTER_NEAREST\n",
    "\n",
    "    x_image, y_image = process_image(image, mask)\n",
    "    gt = y_image.squeeze()\n",
    "\n",
    "    ## Oracle\n",
    "    for j, masked_out in enumerate(masks):\n",
    "\n",
    "        masked_out = ~masked_out\n",
    "\n",
    "        # dice_remain = dice_metric(gt[~masked_out], prediction[~masked_out])\n",
    "        # est_dice_remain = estimate_dice_li(predictions_arr[0, img_idx,:,:][~masked_out])\n",
    "        # pos_remain.append((predictions_arr[0, img_idx, :,:][~masked_out] > 0.5).mean())\n",
    "\n",
    "        dice_remain = estimation_info[selection_idx]['true dice'][j]\n",
    "        #est_dice_remain = estimation_info[selection_idx]['est dice'][j]\n",
    "        est_dice_remain = estimates_matrix_ts[selection_idx, j]\n",
    "        pos_remain.append(estimation_info[selection_idx]['positive remaining'][j])\n",
    "        \n",
    "        est_dice_remain_list.append(est_dice_remain)\n",
    "        dice_remain_list.append(dice_remain)\n",
    "\n",
    "        w_remain = (gt[~masked_out].sum() + prediction[~masked_out].sum()) / (gt.sum() + prediction.sum())\n",
    "        w_oracle = 1 - w_remain\n",
    "        true_w.append(w_remain)\n",
    "\n",
    "        est_w_remain = prediction[~masked_out].sum() / prediction.sum()\n",
    "        est_w_oracle = 1 - est_w_remain\n",
    "        est_w.append(est_w_remain)\n",
    "\n",
    "        dsc_reanno = w_remain * dice_remain + w_oracle * 1\n",
    "        est_dsc_reanno = est_w_remain * est_dice_remain + est_w_oracle * 1\n",
    "\n",
    "        dsc_curve.append(dsc_reanno)\n",
    "        est_dsc_curve.append(est_dsc_reanno)\n",
    "\n",
    "    ## Random Patches\n",
    "    # convolve a random image because we require the references points which are\n",
    "    # outputted by convolve_uncertainty\n",
    "\n",
    "    \n",
    "    # Select Random Point within Circle\n",
    "    dists = [dist2center(ref) for ref in convolution_dict['reference']]\n",
    "    # remove those which are further away than radius\n",
    "    dist_constraint = np.array(dists) > 510/2\n",
    "    population = np.array(convolution_dict['reference'])[dist_constraint, :]\n",
    "    random_references = population[np.random.randint(population.shape[0], size=5), :]\n",
    "\n",
    "    masked_out = np.zeros_like(prediction).astype(bool)\n",
    "    random_dsc_curve = []\n",
    "    random_dice_remain_list = []\n",
    "\n",
    "    for i in range(len(random_references) + 1):\n",
    "\n",
    "        dice_remain = dice_metric(gt[~masked_out], prediction[~masked_out])\n",
    "        random_dice_remain_list.append(dice_remain)\n",
    "\n",
    "        w_remain = (gt[~masked_out].sum() + prediction[~masked_out].sum()) / (gt.sum() + prediction.sum())\n",
    "        w_oracle = 1 - w_remain\n",
    "        \n",
    "        est_w_remain = prediction[~masked_out].sum() / prediction.sum()\n",
    "        est_w_oracle = 1 - est_w_remain\n",
    "        \n",
    "        dsc_reanno = w_remain * dice_remain + w_oracle * 1\n",
    "        \n",
    "        random_dsc_curve.append(dsc_reanno)\n",
    "        \n",
    "        # update mask\n",
    "        if i <= len(random_references) - 1:\n",
    "            patch_ref = random_references[i]\n",
    "            masked_out[patch_ref[0]:patch_ref[0] + cfg.patch_size, patch_ref[1]:patch_ref[1] + cfg.patch_size] = True\n",
    "\n",
    "\n",
    "    return {'true dsc curve': dsc_curve, 'est dsc curve': est_dsc_curve,\n",
    "            'random curve': random_dsc_curve}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove all objects that are not required at the moment to free RAM\n",
    "required_obs = ['ensemble_dict', 'estimation_info', 'oracle_and_random',\n",
    "                'split_x', 'split_y', 'process_image', 'estimates_matrix_ts',\n",
    "                'dist2center', 'convolution_dict', 'required_obs', 'image_selection_ids',\n",
    "                'cv2', 'np', 'dice_metric', 'cfg', 'config_suffix', 'split_suffix',\n",
    "                'pkl', 'plt', 'ts_scaled_estimate_remain', 'inputs', 'targets',\n",
    "                'run_again', 'container']\n",
    "\n",
    "for name in dir():\n",
    "    if not (name.startswith('_') or name in required_obs):\n",
    "        del globals()[name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('cache/' + config_suffix + '_oracle_results' + split_suffix + '.pkl', 'rb') as file4:\n",
    "    oracle_results = pkl.load(file4)\n",
    "    file4.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if config_suffix == 'all_imgs':\n",
    "    import pandas as pd\n",
    "    oracle_true = np.array(pd.read_csv('cache/all_imgs_oracle_true.csv', header=None))\n",
    "    oracle_est = np.array(pd.read_csv('cache/all_imgs_oracle_est.csv', header=None))\n",
    "    oracle_random = np.array(pd.read_csv('cache/all_imgs_oracle_random.csv', header=None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if run_again: \n",
    "    oracle_results = []\n",
    "\n",
    "    for selection_idx, original_idx in enumerate(image_selection_ids):\n",
    "        oracle_results.append(oracle_and_random(original_idx, selection_idx))\n",
    "\n",
    "    with open('cache/' + config_suffix + '_oracle_results' + split_suffix + '.pkl', 'wb') as file4:\n",
    "        pkl.dump(oracle_results, file4)\n",
    "    file4.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oracle_true = np.array([img_dict['true dsc curve'] for img_dict in oracle_results])\n",
    "oracle_est = np.array([img_dict['est dsc curve'] for img_dict in oracle_results])\n",
    "oracle_random = np.array([img_dict['random curve'] for img_dict in oracle_results])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chest_x-rays",
   "language": "python",
   "name": "chest_x-rays"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}