AL_CV_parameter-tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AL_CV_parameter-tuning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lukekolbe/AL-in-CreditScoring/blob/main/AL_CV_parameter-tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bs2HdyuEA3Pu"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJ4RBjIAgQ_o"
      },
      "source": [
        "!conda install git pip\n",
        "\n",
        "#import git\n",
        "!pip install git+git://github.com/NUAA-AL/alipy.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-r6TwUlA8H5",
        "outputId": "6353d28a-8c2b-47be-f9fe-4a8551572818"
      },
      "source": [
        "### FOR RUNNING ON COLAB:\n",
        "#update scikit-learn and imbalanced-learn to recent version\n",
        "#!pip install sklearn-pandas-transformers\n",
        "#!pip uninstall sklearn-pandas-transformers -y\n",
        "#!pip uninstall sklearn_transformers_pandas -y\n",
        "\n",
        "\n",
        "#!pip uninstall scikit-learn -y\n",
        "#!pip install -U scikit-learn==1.0.1 #specific version, because skopt does not work with sklearn 1.0.0 \n",
        "\n",
        "#pip uninstall imbalanced-learn -y\n",
        "!pip install -U imbalanced-learn\n",
        "\n",
        "!pip install scikit-optimize\n",
        "\n",
        "!pip install git+https://github.com/NUAA-AL/alipy.git\n",
        "\n",
        "#!pip install git+https://github.com/jundongl/scikit-feature\n",
        "#!pip install git+https://github.com/lukekolbe/scikit-feature\n",
        "\n",
        "#!pip install cvxpy"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting git+git://github.com/lukekolbe/scikit-feature\n",
            "  Cloning git://github.com/lukekolbe/scikit-feature to c:\\users\\kolbeluk1\\appdata\\local\\temp\\9\\pip-req-build-cn_ql178\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  Running command git clone -q git://github.com/lukekolbe/scikit-feature 'C:\\Users\\kolbeluk1\\AppData\\Local\\Temp\\9\\pip-req-build-cn_ql178'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsS3enpvA9vz"
      },
      "source": [
        "############ LIBRARIES\n",
        "\n",
        "import os\n",
        "import time\n",
        "import datetime\n",
        "import random\n",
        "import multiprocessing\n",
        "import pickle\n",
        "import re\n",
        "import copy\n",
        "import gc\n",
        "import sys\n",
        "import json\n",
        "gc.enable()\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "pd.set_option('display.max_columns', None)\n",
        "\n",
        "'''import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import cm\n",
        "plt.style.use('default')\n",
        "%matplotlib inline\n",
        "import seaborn as sns'''\n",
        "\n",
        "from itertools import cycle\n",
        "\n",
        "\n",
        "import scipy.stats\n",
        "\n",
        "#from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "#from sklearn.model_selection import StratifiedKFold  ##### what is this used for?\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.metrics import brier_score_loss\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import fbeta_score\n",
        "\n",
        "# FOR CROSS VALIDATED HYPERPARAMETER TUNING\n",
        "# use imblearn pipeline instead of sklearn pipeline to skip AL sampling process in the prediction phase\n",
        "from imblearn.pipeline import Pipeline\n",
        "from imblearn import FunctionSampler\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier \n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "from alipy import ToolBox\n",
        "from alipy import query_strategy\n",
        "from alipy.index import IndexCollection\n",
        "from alipy import data_manipulate\n",
        "import cvxpy\n",
        "\n",
        "############ RANDOMNESS\n",
        "# seed function\n",
        "def seed_everything(seed):\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    \n",
        "# set seed\n",
        "seed = 30\n",
        "seed_everything(seed)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ouyUQiiGgQ_y",
        "outputId": "4ec399af-2236-42b3-f13b-025f37de79ce"
      },
      "source": [
        "os.getcwd()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'C:\\\\Users\\\\kolbeluk1\\\\AL_THESIS'"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lt5v7qnlA_3_"
      },
      "source": [
        "#os.chdir('/gdrive/My Drive/ACTIVE LEARNING THESIS/')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zu_FB9c5BB_v"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cal7cW8gBDw5"
      },
      "source": [
        "# CV helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uQYLaSjfBLdR"
      },
      "source": [
        "# get strategy\n",
        "\n",
        "def strategy_getter(X, y, strategy_name=\"QueryInstanceRandom\", train_idx = None, **kwargs):\n",
        "    \"\"\"Return the query strategy object from alipy package\"\"\"\n",
        "    \n",
        "    try:\n",
        "        exec(\"from alipy.query_strategy import \" + strategy_name)\n",
        "    except:\n",
        "        raise KeyError(\"Strategy \" + strategy_name + \" is not implemented in ALiPy.\")\n",
        "    strategy = None\n",
        "    \n",
        "    if train_idx is not None:\n",
        "      strategy = eval(strategy_name + \"(X=X, y=y, train_idx = train_idx, **kwargs)\")\n",
        "    else:\n",
        "      strategy = eval(strategy_name + \"(X=X, y=y, **kwargs)\")\n",
        "          \n",
        "    # print(strategy)\n",
        "    return strategy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "2W5-ra5R63YA",
        "outputId": "8446ba71-afe4-41ec-d582-f4f1b966a88e"
      },
      "source": [
        "'''def pre_sampler(X=None\n",
        "              , y=None):\n",
        "  \n",
        "  sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "  for label, unlabel in sss.split(X=X, y=y):\n",
        "    label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \n",
        "\n",
        "  return X[label_idx,:], y[label_idx]'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'def pre_sampler(X=None\\n              , y=None):\\n  \\n  sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \\n  for label, unlabel in sss.split(X=X, y=y):\\n    label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \\n\\n  return X[label_idx,:], y[label_idx]'"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rlO4CCnKWF4y"
      },
      "source": [
        "\n",
        "class CustomScaler(BaseEstimator, TransformerMixin):\n",
        "  \n",
        "  def __init__(self, with_centering=True, with_scaling=True, seed=30):\n",
        "        self.seed = seed\n",
        "        self.with_centering = with_centering\n",
        "        self.with_scaling = with_scaling\n",
        "        self.scaler = RobustScaler(with_centering=self.with_centering, with_scaling = self.with_scaling)\n",
        "\n",
        "\n",
        "  #estimator method\n",
        "  def fit(self, X, y):\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=int(0.5*len(X)), random_state=self.seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "    for label, unlabel in sss.split(X=X, y=y):\n",
        "      label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \n",
        "      #print(np.shape(y))\n",
        "      #print(len(X[label_idx,:]))\n",
        "\n",
        "      self.scaler.fit(X[label_idx,:])\n",
        "      return self\n",
        "\n",
        "  #transfprmation\n",
        "  def transform(self, X, y = None):\n",
        "    X = self.scaler.transform(X)\n",
        "    return X\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iFRxIj1rBNFI"
      },
      "source": [
        "# Custom Sampler Class, only run in the fit phase of the pipeline, skipped in predict phase\n",
        "#####################################\n",
        "def AL_resampler(X=None\n",
        "              , y=None\n",
        "              #, key # used to select correct strategy name from AL_Models\n",
        "              , strategy_name = None\n",
        "              , pass_index=False\n",
        "              , model = None\n",
        "              #, skip = False\n",
        "              , seed=30\n",
        "              , **kwargs):\n",
        "\n",
        "  \n",
        "  os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "  random.seed(seed)\n",
        "  np.random.seed(seed)\n",
        "  \n",
        "  # re-create the sample that was used to scale the data!\n",
        "  # this is done so that the scaling is only done with data that is labeled, so that there is no leakage\n",
        "  # this is the intial labeled sample that all strategies share\n",
        "  sss = StratifiedShuffleSplit(n_splits=1, test_size=int(0.5*len(X)), random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "  for label, unlabel in sss.split(X=X, y=y):\n",
        "    label_idx, unlabel_idx = np.asarray(label), np.asarray(unlabel)    \n",
        "        \n",
        "\n",
        "  n_instances = int(0.5*len(unlabel_idx))\n",
        "\n",
        "  '''#if skip and strategy_name != 'Oracle':\n",
        "  if skip:\n",
        "    # for models such as oracle, score, random, no AL parameters need tuning\n",
        "    # Hence a random selection of instances is passed to the classifier for training\n",
        "    # another small sample is drawn from the unlabeled data that is added to the labeled set\n",
        "    # the size of the labeled data is then equivalent to the size of the sample that is curated by AL: 75% of the input data, 50% of which are picked randomly (and scaled with)\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=n_instances, random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "    for label, unlabel in sss.split(X=X[unlabel_idx], y=y[unlabel_idx]):\n",
        "        label_add, unlabel_keep = np.asarray(label), np.asarray(unlabel)\n",
        "    \n",
        "    label_idx = np.concatenate((label_idx, unlabel_idx[label_add]))\n",
        "    unlabel_idx = unlabel_idx[unlabel_keep]\n",
        "\n",
        "  #else:'''\n",
        "  ##############\n",
        "  # initialize AL strategy\n",
        "  ##############\n",
        "  #t_idx = np.arange(len(X))\n",
        "  t_idx = np.concatenate((label_idx, unlabel_idx))\n",
        "\n",
        "  if pass_index and not strategy_name == 'QueryInstanceLAL':\n",
        "    strategy = strategy_getter(X, y, strategy_name, t_idx, **kwargs)\n",
        "\n",
        "  elif not pass_index and not strategy_name == 'QueryInstanceLAL':\n",
        "    strategy = strategy_getter(X, y, strategy_name, **kwargs)\n",
        "\n",
        "  # special case because of extra RF; might be prohibitively expensive to tune\n",
        "  elif strategy_name == 'QueryInstanceLAL':\n",
        "    #provide a path for the generic training data, downloaded from https://github.com/ksenia-konyushkova/LAL/tree/master/lal%20datasets\n",
        "    #otherwise the data will be downloaded in every iteration, which is costly in a CV loop\n",
        "    #data_path = '/gdrive/My Drive/ACTIVE LEARNING THESIS/LAL_trainingdata/'\n",
        "    #file_path = '/gdrive/My Drive/ACTIVE LEARNING THESIS/LAL_trainingdata/LAL-iterativetree-simulatedunbalanced-big.npz'\n",
        "    \n",
        "    param_dict = {**kwargs}\n",
        "    reg_est=param_dict.pop('reg_est', None)\n",
        "    reg_depth=param_dict.pop('reg_depth', None)\n",
        "    reg_feat=param_dict.pop('reg_feat', None)\n",
        "    if reg_feat > np.shape(X)[1]:\n",
        "      reg_feat = np.shape(X)[1]\n",
        "\n",
        "    strategy = strategy_getter(X, y, strategy_name = strategy_name, mode='LAL_iterative', train_slt=False, **param_dict)\n",
        "    starttime = time.time()\n",
        "    strategy.download_data()\n",
        "    print('duration of data download', starttime - time.time())\n",
        "    strategy.train_selector_from_file(reg_est=reg_est, reg_depth=reg_depth, feat=reg_feat)\n",
        "\n",
        "  ##############\n",
        "  # run AL selection once\n",
        "  ##############\n",
        "\n",
        "  #fit model to then pass it as argument to some AL strategies\n",
        "  if not model == None:\n",
        "    model.fit(X[label_idx], y[label_idx])\n",
        "\n",
        "  if strategy_name == 'QueryInstanceQUIRE': #strategy quire has no batch mode\n",
        "    label_idx = IndexCollection(label_idx)\n",
        "    unlabel_idx = IndexCollection(unlabel_idx)\n",
        "\n",
        "    print('starting selection')\n",
        "    for n in range(n_instances):\n",
        "      selection = strategy.select(label_index=label_idx.index, unlabel_index=unlabel_idx.index)[0] # returns a list of len one, hence the [0]\n",
        "      label_idx.update(selection)\n",
        "      unlabel_idx.difference_update(selection)\n",
        "      if n%10 == 0:\n",
        "        print(f\"selected {n} cases\")\n",
        "    label_idx = label_idx.index\n",
        "\n",
        "  elif strategy_name in ['QueryInstanceBMDR', 'QueryInstanceSPAL']:\n",
        "    select_idx = strategy.select(label_idx, unlabel_idx, model=model,batch_size=n_instances, qp_solver = 'OSQP')\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "\n",
        "  elif model == None:\n",
        "    select_idx = strategy.select(IndexCollection(label_idx), IndexCollection(unlabel_idx), batch_size=n_instances)\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "  \n",
        "  elif not model ==None:\n",
        "    select_idx = strategy.select(label_idx, unlabel_idx, model=model, batch_size=n_instances)\n",
        "    label_idx = np.concatenate((label_idx, select_idx))\n",
        "\n",
        "\n",
        "  return X[label_idx,:], y[label_idx]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqI64HfUgQ__",
        "outputId": "c7d84f74-3a5e-41ed-ac29-c42409adf463"
      },
      "source": [
        " 21%20"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "DpZhIYiFIYyH",
        "outputId": "cfc817f2-b729-4130-83b2-796a66de3de3"
      },
      "source": [
        "'''from sklearn.ensemble import IsolationForest\n",
        "\n",
        "def outlier_rejection(X, y, **kwargs):\n",
        "    \"\"\"This will be our function used to resample our dataset.\"\"\"\n",
        "    model = IsolationForest(max_samples = 'auto', contamination='auto', random_state=seed, n_jobs = -1, **kwargs)\n",
        "    model.fit(X)\n",
        "    o_pred = model.predict(X)\n",
        "    print(np.mean(o_pred))\n",
        "    return X[o_pred == 1], y[o_pred == 1], o_pred\n",
        "\n",
        "\n",
        "reject_sampler = FunctionSampler(func=outlier_rejection)\n",
        "X_inliers, y_inliers = reject_sampler.fit_resample(X_train, y_train)'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'from sklearn.ensemble import IsolationForest\\n\\ndef outlier_rejection(X, y, **kwargs):\\n    \"\"\"This will be our function used to resample our dataset.\"\"\"\\n    model = IsolationForest(max_samples = \\'auto\\', contamination=\\'auto\\', random_state=seed, n_jobs = -1, **kwargs)\\n    model.fit(X)\\n    o_pred = model.predict(X)\\n    print(np.mean(o_pred))\\n    return X[o_pred == 1], y[o_pred == 1], o_pred\\n\\n\\nreject_sampler = FunctionSampler(func=outlier_rejection)\\nX_inliers, y_inliers = reject_sampler.fit_resample(X_train, y_train)'"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O3TKLv6JBTln"
      },
      "source": [
        "# Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U4b6VBKDBFRF",
        "outputId": "9f9ebf51-3a8b-4164-f629-4af9426eb77a"
      },
      "source": [
        "############ DATA IMPORT\n",
        "\n",
        "## available datasets\n",
        "# bene1         # some \n",
        "\n",
        "    # OK gmsc          # shape:  (150000, 68)\n",
        "    # OK uk            # very low bad rate, shape:  (30000, 51), y mean:  0.04\n",
        "    # OK lendingclub   # loooow performance, X shape:  (41623, 114) y shape:  (41623,) y mean:  0.1331235134420873    \n",
        "    \n",
        "    # OK bene2         # some learning, shape:  (7190, 28) >> 3 folds NO WASTE\n",
        "    \n",
        "    # bene1_nobins  # shape:  (3123, 18) # 5 folds NO WASTE\n",
        "        # hmeq          # not promising either, shape:  (5960, 20) # 5 folds NO WASTE\n",
        "        # australian    # very small dataset shape:  (690, 42) >> 10 folds NO WASTE\n",
        "        # german        # shape:  (1000, 61) >> 10 folds NO WASTE\n",
        "        # thomas        # loooow performance shape:  (1225, 28) >> 10 folds NO WASTE\n",
        "\n",
        "# pakdd         # shape:  (50000, 373), y mean:  0.26082\n",
        "\n",
        "dataset = \"pakdd\"\n",
        "# C:\\\\Users\\\\kolbeluk1\\\\AL_THESIS\n",
        "df = pd.read_csv('C:\\\\Users\\\\kolbeluk1\\\\AL_THESIS\\\\prepared_data\\\\{}.csv'.format(dataset))\n",
        "#df = pd.read_csv('/gdrive/My Drive/ACTIVE LEARNING THESIS/prepared_data/{}.csv'.format(dataset))\n",
        "#df = pd.read_csv('/gdrive/My Drive/ACTIVE LEARNING THESIS/prepared_data/bene2.csv')\n",
        "\n",
        "# remove NA\n",
        "df = df.dropna()\n",
        "df.reset_index(drop = True, inplace = True)\n",
        "\n",
        "#print(df)\n",
        "# extract label\n",
        "df['BAD'][df['BAD']=='BAD']  = 1\n",
        "df['BAD'][df['BAD']=='GOOD'] = 0\n",
        "df['BAD'] = df['BAD'].astype('int')\n",
        "\n",
        "\n",
        "y_temp = df['BAD']\n",
        "del df['BAD']\n",
        "\n",
        "#one hot encoding\n",
        "df = pd.get_dummies(df)\n",
        "\n",
        "#transform to numpy array >> same location for df and X\n",
        "X = df.to_numpy()\n",
        "y = y_temp.to_numpy()\n",
        "\n",
        "print(\"X type: \", type(X), \"X shape: \",X.shape,\"y shape: \", y.shape, \"y mean: \", np.mean(y))\n",
        "print (id(X), id(df))\n",
        "#print(y)\n",
        "\n",
        "# append_record: helper function that adds best-parameter for every model to dict and saves it\n",
        "filename = f'{dataset}_tuned-params'\n",
        "\n",
        "def append_record(record):\n",
        "    with open(f'{filename}', 'a') as f:\n",
        "        json.dump(record, f)\n",
        "        f.write(os.linesep)\n",
        "filename"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X type:  <class 'numpy.ndarray'> X shape:  (50000, 373) y shape:  (50000,) y mean:  0.26082\n",
            "763698962576 763701202176\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "'pakdd_tuned-params'"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NqiO1UexBVTr"
      },
      "source": [
        "# Split off Validation Set\n",
        "\n",
        "implement scheme for splitting a validation set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lbD-fnJ5Bc5w",
        "outputId": "0e0f4134-157d-4317-e62f-bbd548149009"
      },
      "source": [
        "if len(y) > 25000:\n",
        "    validation_size = 5000\n",
        "    folds = 3\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=validation_size, random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "    for model, validation in sss.split(X=X, y=y):\n",
        "        model_idx, validation_idx = np.asarray(model), np.asarray(validation)    \n",
        "\n",
        "    print(np.sum(model_idx), np.sum(validation_idx))\n",
        "    X_val, y_val = X[validation_idx,:], y[validation_idx]\n",
        "    #X_model, y_model = X[model_idx,:], y[model_idx]\n",
        "\n",
        "elif len(y) < 2000:\n",
        "    folds = 10\n",
        "    X_val, y_val = X, y\n",
        "\n",
        "else:\n",
        "    folds = 3\n",
        "    X_val, y_val = X, y\n",
        "    \n",
        "print(np.shape(X), np.shape(X_val), 'folds: ', folds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1126541891 123433109\n",
            "(50000, 373) (5000, 373) folds:  3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jIpOZHE4gRAC",
        "outputId": "4ccc6ba0-e8aa-49d1-e3b6-64db62eb942b"
      },
      "source": [
        "#hmeq\n",
        "\n",
        "if len(y) > 5000:\n",
        "    validation_size = 3000\n",
        "    folds = 3\n",
        "    sss = StratifiedShuffleSplit(n_splits=1, test_size=validation_size, random_state=seed) # THESE ARE POSITIONS, NOT INDICES    \n",
        "    for model, validation in sss.split(X=X, y=y):\n",
        "        model_idx, validation_idx = np.asarray(model), np.asarray(validation)    \n",
        "\n",
        "    print(np.sum(model_idx), np.sum(validation_idx))\n",
        "    X_val, y_val = X[validation_idx,:], y[validation_idx]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "8948729 8809091\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WkNXYuN1CPs7"
      },
      "source": [
        "# CV Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gV0fQCgxCRIZ"
      },
      "source": [
        "# use imblearn pipeline to skip AL sampling process in the prediction phase\n",
        "CLF_pipe = Pipeline(steps=[#('outlier', FunctionSampler(func=outlier_rejection)),\n",
        "                      ('scaler', CustomScaler(seed=seed)),\n",
        "                      ('AL_sampler', FunctionSampler(func=AL_resampler)),    # this will trigger a call to __init__\n",
        "                      ('clf', LogisticRegression(random_state=seed))])\n",
        "\n",
        "#cv_parameters_random = {'random': [{'strategy_name': 'QueryInstanceRandom'}]}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EwWdWAqXUJn_",
        "outputId": "634d3871-3f8f-4686-854a-fbfee9e6d0b4"
      },
      "source": [
        "## RUN TUNING FOR CLASSIFIER FIRST!\n",
        "\n",
        "for key in ['random']:\n",
        "\n",
        "  param_grid={'clf__solver': ['liblinear'], \n",
        "              'clf__penalty': ['l1', 'l2'], \n",
        "              'clf__C': [0.001,0.01,0.1,1.,10.,100.], \n",
        "              'clf__max_iter': [50, 100, 250, 500], \n",
        "              'clf__tol': [0.0001, 0.001, 0.0001, 0.00001],               \n",
        "              #'clf__class_weight':['balanced', None], \n",
        "              'AL_sampler__kw_args': [{'strategy_name': 'QueryInstanceRandom'}]} #for  x in [125, 250, 500, 600], 'outlier__kw_args': transform_parameters, \n",
        "\n",
        "  grid_search_clf = RandomizedSearchCV(CLF_pipe, param_distributions=param_grid, n_iter = 500, cv=10, n_jobs=20, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  grid_search_clf.fit(X_val, y_val)\n",
        "\n",
        "  #### PRINT RESULTS\n",
        "  '''print(\"Best parameters set found on development set:\", \"\\n\")\n",
        "  print(grid_search_clf.best_params_)\n",
        "  print(\"Grid scores on development set:\", \"\\n\")\n",
        "  means_roc = grid_search_clf.cv_results_['mean_test_roc_auc']\n",
        "  stds_roc = grid_search_clf.cv_results_['std_test_roc_auc']\n",
        "  for means_roc, stds_roc, params in zip(means_roc, stds_roc, grid_search_clf.cv_results_['params']):\n",
        "      print(\"%0.3f (+/-%0.03f) for %r\"\n",
        "            % (means_roc, stds_roc * 2, params))\n",
        "\n",
        "  print(\"\\n\", \"Detailed classification report:\", \"\\n\")\n",
        "  print(\"The model is trained on the full development set.\", \"\\n\")\n",
        "  print(\"The scores are computed on the full evaluation set.\", \"\\n\")\n",
        "  y_pred = grid_search_clf.predict(X_model)\n",
        "  print(classification_report(y_model, y_pred))'''\n",
        "\n",
        "  ### SAVE RESULTS\n",
        "  AL_params = copy.deepcopy(grid_search_clf.best_params_['AL_sampler__kw_args'])   #[i] for i in grid_search_clf.best_params_['sampler__kw_args'] if i not in ['key', 'name', 'sample_size']]\n",
        "  AL_params.pop('pass_index', None)\n",
        "  AL_params.pop('model', None)\n",
        "  AL_params.pop('skip', None)\n",
        "\n",
        "  CLF_params = copy.deepcopy(grid_search_clf.best_params_)\n",
        "  CLF_params.pop('AL_sampler__kw_args', None)\n",
        "  CLF_params.pop('outlier__kw_args', None)\n",
        "\n",
        "  for clf_key in list(CLF_params.keys()):\n",
        "    new_key = re.sub(r'clf__', '', clf_key)\n",
        "    CLF_params[new_key] = CLF_params.pop(clf_key)\n",
        "\n",
        "  cv_param_dict = {f'{key}': {#'outlier_rf': outlier_params,\n",
        "                 'AL': AL_params,\n",
        "                 'CLF': CLF_params}}\n",
        "  \n",
        "  append_record(cv_param_dict)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 10 folds for each of 192 candidates, totalling 1920 fits\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IEWUJKS-Ulvb",
        "outputId": "a4d6221c-4353-4b10-a293-ce1533c3a206"
      },
      "source": [
        "clf = LogisticRegression(random_state=seed)\n",
        "\n",
        "clf_tuned_params = copy.deepcopy(grid_search_clf.best_params_)\n",
        "clf_tuned_params.pop('AL_sampler__kw_args', None)\n",
        "\n",
        "for clf_key in list(clf_tuned_params.keys()):\n",
        "  new_key = re.sub(r'clf__', '', clf_key)\n",
        "  clf_tuned_params[new_key] = clf_tuned_params.pop(clf_key)\n",
        "\n",
        "clf.set_params(**clf_tuned_params)\n",
        "print(str(clf))\n",
        "\n",
        "AL_pipe = Pipeline(steps=[#('outlier', FunctionSampler(func=outlier_rejection)),\n",
        "                      ('scaler', CustomScaler(seed=seed)),\n",
        "                      ('AL_sampler', FunctionSampler(func=AL_resampler)),    # this will trigger a call to __init__\n",
        "                      ('clf', clf)])\n",
        "\n",
        "cv_parameters = {'unc': [{'strategy_name': 'QueryInstanceUncertainty', 'model': clf, 'measure': m} for m in ['entropy', 'least_confident', 'margin', 'distance_to_boundary']]\n",
        "                ,'qbc': [{'strategy_name': 'QueryInstanceQBC', 'model': clf, 'method': 'query_by_bagging', 'disagreement': d} for d in ['vote_entropy', 'KL_divergence']]\n",
        "                #,'eer': [{'strategy_name': 'QueryExpectedErrorReduction', 'model': clf}] #slow, even without tuning >> maybe don't tune at all? See what the other tuning cycles choose as clf parameters?\n",
        "                ,'dw': [{'strategy_name': 'QueryInstanceDensityWeighted', 'model': clf, 'uncertainty_meansure': u , 'distance': d, 'beta': b} for u in ['least_confident', 'margin', 'entropy'] for d in ['cityblock', 'cosine', 'euclidean'] for b in [0.5, 1, 2]]\n",
        "                ,'density':[{'strategy_name': 'QueryInstanceGraphDensity', 'pass_index': True, 'metric': m} for m in ['canberra', 'jaccard', 'cosine', 'hamming']]\n",
        "                ,'cors' :  [{'strategy_name': 'QueryInstanceCoresetGreedy', 'pass_index': True, 'distance': d} for d in ['cityblock', 'cosine', 'euclidean']]\n",
        "                ,'lal': [{'strategy_name': 'QueryInstanceLAL', 'cls_est': cls_est, 'reg_est': reg_est, 'reg_depth': reg_depth, 'reg_feat': reg_feat} for cls_est in [16,32,64] for reg_est in [32,64,128] for reg_depth in [5,10,20] for reg_feat in [5,6,7]] # SLOOOW to tune\n",
        "                #,'quire': [{'strategy_name': 'QueryInstanceQUIRE','pass_index': True, 'kernel': 'rbf', 'lambda': l, 'gamma': g} for g in [0.1, 1, 10] for l in [0.1, 1, 10]] #SLOOOOOOOOW, 40 mins at njobs = 1, cv = 2, n_iter = 25 >> extremely low spec tuning\n",
        "                #,'bmdr': [{'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': b, 'gamma': g, 'rho': r, 'gamma_ker': gk} for b in [100, 1000, 10000] for g in [0.01, 0.1, 1] for r in [0.1, 1, 10] for gk in [0.1, 1, 10]] # issues with the solver ECOS; super slowto tune\n",
        "                #,'spal': [{'strategy_name': 'QueryInstanceSPAL', 'kernel': 'rbf', 'mu': mu, 'gamma': g, 'rho': r, 'lambda_init': li, 'lambda_pace': lp, 'gamma_ker': gk} for mu in [0.01, 0.1, 1] for g in [0.01, 0.1, 1] for r in [0.1, 1, 10] for li in [0.01, 0.1, 1] for lp in [0.001, 0.01, 0.1] for gk in [0.1, 1, 10]]                \n",
        "                ,'spal': [{'strategy_name': 'QueryInstanceSPAL', 'kernel': 'rbf', 'mu': mu, 'gamma': g,'rho': r, 'lambda_init': li, 'lambda_pace': lp, } for mu in [0.01, 0.1, 1] for g in [0.01, 0.1, 1] for r in [0.1, 1, 10] for li in [0.01, 0.1, 1] for lp in [0.001, 0.01, 0.1]]                \n",
        "                ,'bmdr': [{'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': b, 'gamma': g, 'rho': r} for b in [100, 1000, 10000] for g in [0.01, 0.1, 1] for r in [0.1, 1, 10]] # issues with the solver ECOS; super slowto tune\n",
        "                }"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "LogisticRegression(C=0.1, max_iter=50, random_state=30, solver='liblinear',\n",
            "                   tol=1e-05)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5s7VjzPHgRAE",
        "outputId": "6c9a9b9f-f36f-4e01-939e-c5bf21000d6f"
      },
      "source": [
        "#import time\n",
        "\n",
        "start = time.time()\n",
        "print(start)\n",
        "#'unc', 'qbc', 'dw', 'cors', 'density', 'bmdr', 'spal', 'eer', 'lal'\n",
        "#for key in ['spal', 'bmdr']: \n",
        "for key in cv_parameters.keys(): #eer, quire, lal, spal, bmdr\n",
        "#for key in ['bmdr']:\n",
        "  loop_start = time.time()\n",
        "  #iter_start= time.time()\n",
        "  param_grid={'AL_sampler__kw_args': cv_parameters[key]} #for  x in [125, 250, 500, 600], 'outlier__kw_args': transform_parameters, \n",
        "  \n",
        "  grid_search_AL = RandomizedSearchCV(AL_pipe, param_distributions=param_grid, n_iter=150, cv=folds, n_jobs=18, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  #grid_search_AL = GridSearchCV(AL_pipe, param_grid=param_grid, cv=folds, n_jobs=20, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  grid_search_AL.fit(X_val, y_val)\n",
        "\n",
        "  AL_params = copy.deepcopy(grid_search_AL.best_params_['AL_sampler__kw_args'])   #[i] for i in grid_search_AL.best_params_['sampler__kw_args'] if i not in ['key', 'name', 'sample_size']]\n",
        "  AL_params.pop('pass_index', None)\n",
        "  AL_params.pop('model', None)\n",
        "  AL_params.pop('skip', None)\n",
        "\n",
        "  CLF_params = copy.deepcopy(grid_search_AL.best_params_)\n",
        "  CLF_params.pop('AL_sampler__kw_args', None)\n",
        "  #CLF_params.pop('outlier__kw_args', None)\n",
        "\n",
        "  for clf_key in list(CLF_params.keys()):\n",
        "    new_key = re.sub(r'clf__', '', clf_key)\n",
        "    CLF_params[new_key] = CLF_params.pop(clf_key)\n",
        "\n",
        "  cv_param_dict = {f'{key}': {#'outlier_rf': outlier_params,\n",
        "                 'AL': AL_params,\n",
        "                 'CLF': CLF_params}}\n",
        "  \n",
        "  append_record(cv_param_dict)\n",
        "  \n",
        "\n",
        "  '''iter_end = time.time()\n",
        "  print(f'time {key}:',iter_end - iter_start)'''\n",
        "  print(f'{key} time:',(time.time() - loop_start)/3600)\n",
        "print('total time:',(time.time() - start)/3600)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1637874726.6753042\n",
            "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
            "unc time: 0.00022953901025984022\n",
            "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
            "qbc time: 0.0004285077253977458\n",
            "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
            "dw time: 0.0011868402030732897\n",
            "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
            "density time: 0.015081219209565056\n",
            "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
            "cors time: 0.009375002185503642\n",
            "Fitting 3 folds for each of 81 candidates, totalling 243 fits\n",
            "duration of data download 0.0\n",
            "Building lal regression model from .\\LAL-iterativetree-simulatedunbalanced-big.npz\n",
            "Done!\n",
            "Oob score =  0.190898747395976\n",
            "lal time: 0.16766286995675828\n",
            "Fitting 3 folds for each of 150 candidates, totalling 450 fits\n",
            "spal time: 0.8833321718374888\n",
            "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
            "bmdr time: 10.521747084061305\n",
            "total time: 11.599044068190787\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ThsKNJiRPwhj",
        "outputId": "c587000b-df73-46a2-cdeb-efbbdedbb83a"
      },
      "source": [
        "#UK\n",
        "\n",
        "start = time.time()\n",
        "print(start)\n",
        "#'unc', 'qbc', 'dw', 'cors', 'density', 'bmdr', 'spal', 'eer', 'lal'\n",
        "#for key in ['spal', 'bmdr']: \n",
        "for key in cv_parameters.keys(): #eer, quire, lal, spal, bmdr\n",
        "#for key in ['bmdr']:\n",
        "  loop_start = time.time()\n",
        "  #iter_start= time.time()\n",
        "  param_grid={'AL_sampler__kw_args': cv_parameters[key]} #for  x in [125, 250, 500, 600], 'outlier__kw_args': transform_parameters, \n",
        "  \n",
        "  grid_search_AL = RandomizedSearchCV(AL_pipe, param_distributions=param_grid, n_iter=150, cv=folds, n_jobs=-1, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  #grid_search_AL = GridSearchCV(AL_pipe, param_grid=param_grid, cv=folds, n_jobs=20, verbose=5, error_score=\"raise\", refit='roc_auc', scoring=['roc_auc'], random_state=seed)\n",
        "  grid_search_AL.fit(X_val, y_val)\n",
        "\n",
        "  AL_params = copy.deepcopy(grid_search_AL.best_params_['AL_sampler__kw_args'])   #[i] for i in grid_search_AL.best_params_['sampler__kw_args'] if i not in ['key', 'name', 'sample_size']]\n",
        "  AL_params.pop('pass_index', None)\n",
        "  AL_params.pop('model', None)\n",
        "  AL_params.pop('skip', None)\n",
        "\n",
        "  CLF_params = copy.deepcopy(grid_search_AL.best_params_)\n",
        "  CLF_params.pop('AL_sampler__kw_args', None)\n",
        "  #CLF_params.pop('outlier__kw_args', None)\n",
        "\n",
        "  for clf_key in list(CLF_params.keys()):\n",
        "    new_key = re.sub(r'clf__', '', clf_key)\n",
        "    CLF_params[new_key] = CLF_params.pop(clf_key)\n",
        "\n",
        "  cv_param_dict = {f'{key}': {#'outlier_rf': outlier_params,\n",
        "                 'AL': AL_params,\n",
        "                 'CLF': CLF_params}}\n",
        "  \n",
        "  append_record(cv_param_dict)\n",
        "  \n",
        "\n",
        "  '''iter_end = time.time()\n",
        "  print(f'time {key}:',iter_end - iter_start)'''\n",
        "  print(f'{key} time:',(time.time() - loop_start)/3600)\n",
        "print('total time:',(time.time() - start)/3600)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1637422400.4788964\n",
            "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
            "unc time: 8.786307440863715e-05\n",
            "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
            "qbc time: 0.0002318920029534234\n",
            "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
            "dw time: 0.0007946791251500448\n",
            "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
            "density time: 0.01941726697815789\n",
            "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
            "cors time: 0.003652158644464281\n",
            "Fitting 3 folds for each of 81 candidates, totalling 243 fits\n",
            "duration of data download 0.0\n",
            "Building lal regression model from .\\LAL-iterativetree-simulatedunbalanced-big.npz\n",
            "Done!\n",
            "Oob score =  0.07399072928952011\n",
            "lal time: 0.1668249283896552\n",
            "Fitting 3 folds for each of 150 candidates, totalling 450 fits\n",
            "spal time: 29.830344127284157\n",
            "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
            "bmdr time: 43.33098174055417\n",
            "total time: 73.35233521223068\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mnetRdFaH-pj",
        "outputId": "e95358b4-6a94-417c-8171-9cc4d1f31b7a"
      },
      "source": [
        "#### PRINT RESULTS\n",
        "print(\"Best parameters set found on development set:\", \"\\n\")\n",
        "print(grid_search_AL.best_params_)\n",
        "print(\"Grid scores on development set:\", \"\\n\")\n",
        "means_roc = grid_search_AL.cv_results_['mean_test_roc_auc']\n",
        "stds_roc = grid_search_AL.cv_results_['std_test_roc_auc']\n",
        "for means_roc, stds_roc, params in zip(means_roc, stds_roc, grid_search_AL.cv_results_['params']):\n",
        "  print(\"%0.3f (+/-%0.03f) for %r\"\n",
        "        % (means_roc, stds_roc * 2, params))\n",
        "\n",
        "print(\"\\n\", \"Detailed classification report:\", \"\\n\")\n",
        "print(\"The model is trained on the full development set.\", \"\\n\")\n",
        "print(\"The scores are computed on the full evaluation set.\", \"\\n\")\n",
        "y_pred = grid_search_AL.predict(X_model)\n",
        "print(classification_report(y_model, y_pred))\n",
        "\n",
        "#outlier_params = copy.deepcopy(grid_search_AL.best_params_['outlier__kw_args'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Best parameters set found on development set: \n",
            "\n",
            "{'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.01, 'rho': 0.1}}\n",
            "Grid scores on development set: \n",
            "\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.01, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.01, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.01, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 0.1, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 100, 'gamma': 1, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.01, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.01, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.01, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 0.1, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 1000, 'gamma': 1, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.01, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.01, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.01, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 0.1, 'rho': 10}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 1, 'rho': 0.1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 1, 'rho': 1}}\n",
            "0.722 (+/-0.122) for {'AL_sampler__kw_args': {'strategy_name': 'QueryInstanceBMDR', 'kernel': 'rbf', 'beta': 10000, 'gamma': 1, 'rho': 10}}\n",
            "\n",
            " Detailed classification report: \n",
            "\n",
            "The model is trained on the full development set. \n",
            "\n",
            "The scores are computed on the full evaluation set. \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.74      0.85      0.79      1665\n",
            "           1       0.58      0.41      0.48       833\n",
            "\n",
            "    accuracy                           0.70      2498\n",
            "   macro avg       0.66      0.63      0.64      2498\n",
            "weighted avg       0.69      0.70      0.69      2498\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9N93weMke8B",
        "outputId": "41edf64a-9131-4fd0-b350-50f95579606d"
      },
      "source": [
        "with open(filename) as f:\n",
        "    param_list = [json.loads(line) for line in f]\n",
        "\n",
        "param_dict = {}\n",
        "for i in range(len(param_list)):\n",
        "  strategy_short = list(param_list[i].keys())[0]\n",
        "  param_dict[strategy_short] = param_list[i][list(param_list[i].keys())[0]]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "ename": "JSONDecodeError",
          "evalue": "Expecting value: line 2 column 1 (char 1)",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
            "\u001b[1;32m<ipython-input-31-844aaa3380cb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mparam_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mparam_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m<ipython-input-31-844aaa3380cb>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mparam_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mparam_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AL_THESIS\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    355\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 357\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    358\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    359\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AL_THESIS\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    335\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    336\u001b[0m         \"\"\"\n\u001b[1;32m--> 337\u001b[1;33m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    338\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    339\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AL_THESIS\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m    353\u001b[0m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    354\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 355\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    356\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;31mJSONDecodeError\u001b[0m: Expecting value: line 2 column 1 (char 1)"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qZC23S9iz4r0"
      },
      "source": [
        "# MISC STUFF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OaPAnzC_YwBO",
        "outputId": "49bc913c-0630-4a80-c398-8fbcbe6c3c91"
      },
      "source": [
        "len(X_val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2000"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXU9KMs8cvMf",
        "outputId": "c930a4de-8f15-45a0-c883-826388a18b88"
      },
      "source": [
        "xn, yn, outliers = outlier_rejection(X_val, y_val, n_estimators=100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.7508\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EJxiivsdgIdV",
        "outputId": "f1f7e8a2-5621-4b1e-cd44-b7da9d748bd2"
      },
      "source": [
        "cs = CustomScaler(with_centering= True, with_scaling=True, seed=30)\n",
        "cs.fit(X_val, y_val)\n",
        "\n",
        "X_test = cs.transform(np.array(X_val))\n",
        "\n",
        "print(X_val[0,:])\n",
        "print('\\n', X_test[0,:])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1438,)\n",
            "719\n",
            "[     0   6437  10355      0      0  27980      6      1      1 225001\n",
            "      3     42      0  37000      3      2      1      3     26     26\n",
            "      4      1      1      1      0     93  37000   2583]\n",
            "\n",
            " [ 0.         -0.44991428  0.08335666  0.          0.          2.54933941\n",
            "  0.          0.          1.         -0.23970915  0.          0.33333333\n",
            "  0.         -0.41101521  0.          0.          0.         -0.375\n",
            " -0.5        -0.38095238 -0.16666667  0.          0.          0.\n",
            "  0.          3.88235294 -0.50818029 -1.36208538]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LKoSDVctjn_I",
        "outputId": "2c833f6b-3680-4305-a0f0-e7dc393f66eb"
      },
      "source": [
        "2.79800e+04"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "27980.0"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "42wj_HRuPjIB"
      },
      "source": [
        "scaler = RobustScaler(with_centering=True, with_scaling=True)\n",
        "scaler2 = RobustScaler(with_centering=True, with_scaling=True)\n",
        "\n",
        "scaler.fit(X_val)\n",
        "X_t = scaler.transform(np.array(X_val)) #scaled version of full dataset\n",
        "\n",
        "scaler2.fit(X_t)\n",
        "X_t2 = scaler2.transform(np.array(X_t)) #scaled version of full dataset\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tx7DUXAodKmR",
        "outputId": "4fc0c9d4-140c-4b65-9110-0a9842df923f"
      },
      "source": [
        "'''from sklearn.model_selection import StratifiedKFold\n",
        "skf = StratifiedKFold(n_splits=5)\n",
        "#X_cv_train, y_cv_splits = skf.split(X_cv, y_cv)\n",
        "indices = []\n",
        "for train_index, test_index in skf.split(X_val, y_val):\n",
        "  #print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
        "  indices.append((train_index, test_index))\n",
        "\n",
        "for i in range(5):\n",
        "  tr_idx = indices[i][0]\n",
        "  ts_idx = indices[i][1]\n",
        "\n",
        "print(len(tr_idx), len(ts_idx))'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "4000 1000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tQ3J30FnpOy-",
        "outputId": "81aa8b09-985d-4f27-fb18-c003afff327f"
      },
      "source": [
        "np.shape(indices)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(5, 2)"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    }
  ]
}