Experiments Execution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from tqdm.notebook import tqdm_notebook\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# My packages\n",
    "from source import parse_mxml as pm\n",
    "from source import log_representation as lr\n",
    "from source import plots as plts\n",
    "from source import drift_detection as dd\n",
    "from source import drift_localization as dl\n",
    "from source import offline_streaming_clustering as off_sc\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from joblib import Parallel, delayed, parallel_backend\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "\n",
    "import os\n",
    "import glob\n",
    "\n",
    "import gc\n",
    "gc.enable()\n",
    "\n",
    "from scipy.spatial import distance\n",
    "from sklearn.base import clone as sk_clone \n",
    "\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def insensitive_glob(pattern):\n",
    "    def either(c):\n",
    "        return '[%s%s]' % (c.lower(), c.upper()) if c.isalpha() else c\n",
    "    return glob.glob(''.join(map(either, pattern)))\n",
    "\n",
    "def if_any(string, lista):\n",
    "    # If the string contains any of the values\n",
    "    # from the list 'lista'\n",
    "    for l in lista:\n",
    "        if l in string:\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List log files\n",
    "logs = insensitive_glob(\"../process_mining_datasets/*/*k.MXML\")\n",
    "logs = [x.replace('..\\\\', '../') for x in logs if \"2.5\" not in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reference objects and map them to strings in dict \n",
    "# used in further methods\n",
    "objects = {\n",
    "    \"model\": {\n",
    "        \"kmeans__k=6\": KMeans(n_clusters=6, random_state=42),\n",
    "        \"kmeans__k=3\": KMeans(n_clusters=3, random_state=42),\n",
    "        \"kmeans__k=2\": KMeans(n_clusters=2, random_state=42),\n",
    "    },\n",
    "    \n",
    "    \"representation\": {\n",
    "        \"binary\": lambda x: lr.get_binary_representation(lr.get_traces_as_tokens(x)),\n",
    "        \"frequency\": lambda x: lr.get_frequency_representation(lr.get_traces_as_tokens(x)),\n",
    "        \"transitions\": lambda x: lr.get_binary_transitions_representation(lr.get_traces_as_tokens(x))\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change patterns and they supported representations\n",
    "\n",
    "binary_support = [\"cb\", \"cf\",  \"cm\", \"rp\", \"pm\", \"fr\"]\n",
    "frequency_support = binary_support + ['lp', 'cp', ]\n",
    "transitions_support = frequency_support + [\"pl\", \"cd\", \"sw\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline Offline Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_PATH = \"../LoanApplications_Offline/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_file_and_run_clustering_pipeline(args, return_result=False):\n",
    "    \"\"\"\n",
    "    Read an event log file, represent it into a feature vector space and\n",
    "    run the trace clustering method over windows. This method outputs results\n",
    "    as gzip csv files into the \"OUTPUT_PATH\" folder, or return the result \n",
    "    as DataFrame when return_result = True.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "        args (dict): Dictionary with the parameters and the log_file path\n",
    "            requiring the following keys:\n",
    "                example = {\n",
    "                    'log': <PATH TO LOG_FILE>,\n",
    "                    'representation': <KEY TO REPRESENTATIONS IN 'objects'>,\n",
    "                    'parameters': [{\n",
    "                        'model': <KEY TO MODEL IN 'objects'>, \n",
    "                        'sliding_window': <WHETHER TO USE SLIDING WINDOW>,\n",
    "                        'window_size': <SIZE OF TRACE WINDOW TO USE>,\n",
    "                        'sliding_step': <STEP OF SLIDING WINDOW>\n",
    "                    }\n",
    "        return_result (bool): Whether to return the result as DataFrame\n",
    "            \n",
    "    \"\"\"\n",
    "    \n",
    "    # Treat file name to structure size and log type\n",
    "    split = args[\"log\"].split(\"\\\\\")\n",
    "\n",
    "    # Parse change pattern name\n",
    "    cd_name = split[1]\n",
    "    log_name = split[2][:-5]\n",
    "\n",
    "    # Parse size of the event_log\n",
    "    log_size = log_name.replace(cd_name, \"\").replace(\"new_\", \"\")\n",
    "    log_size = int(float(log_size.replace(\"k\", \"\")) * 1000)\n",
    "    \n",
    "    # Set up true drifts indexes\n",
    "    y_true = [x for x in range(int(log_size/10), log_size, int(log_size/10))]\n",
    "    \n",
    "    try:\n",
    "        # Read log and apply trace representation technique\n",
    "        log_read = pm.all_prep(open(args[\"log\"]))\n",
    "        df = objects[\"representation\"][args[\"representation\"]](\n",
    "            log_read\n",
    "        )\n",
    "        \n",
    "        col_names = df.columns\n",
    "        \n",
    "        for p in args[\"parameters\"]:\n",
    "            # String to identify results when exporting files\n",
    "            tipo_mudanca = cd_name.replace(\"new_\", \"\")\n",
    "            \n",
    "            cached_info = \"_\".join([\n",
    "                tipo_mudanca,\n",
    "                str(log_size),\n",
    "                p[\"model\"],\n",
    "                args[\"representation\"],\n",
    "                str(p[\"window_size\"]),\n",
    "                str(p[\"sliding_window\"])\n",
    "            ])\n",
    "            \n",
    "            print(cached_info)\n",
    "            \n",
    "            # If already exists, return if needed\n",
    "            file_to_export = OUTPUT_PATH + tipo_mudanca + '/' + cached_info + '.pickle.gzip'\n",
    "            if os.path.exists(file_to_export):\n",
    "                if return_result:\n",
    "                    r_ = pd.read_pickle(\n",
    "                        file_to_export,\n",
    "                        compression='gzip'\n",
    "                    )\n",
    "                    return r_\n",
    "                else:\n",
    "                    continue\n",
    "            \n",
    "            # If file does not exists, run trace clustering step and export file\n",
    "            all_metrics = off_sc.run_offline_clustering_window(\n",
    "                sk_clone(objects[\"model\"][p[\"model\"]]),\n",
    "                p[\"window_size\"],\n",
    "                df,\n",
    "                p[\"sliding_window\"],\n",
    "                sliding_step=p['sliding_step']\n",
    "            )\n",
    "            \n",
    "            if return_result:\n",
    "                return all_metrics\n",
    "            else:\n",
    "                try:\n",
    "                    os.makedirs(OUTPUT_PATH + tipo_mudanca + '/')\n",
    "                except:\n",
    "                    pass\n",
    "                all_metrics.to_pickle(file_to_export, compression=\"gzip\")\n",
    "\n",
    "            gc.collect()\n",
    "    except Exception as e:\n",
    "        raise e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run pipeline for specific case(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fr_5000_kmeans__k=3_binary_125_False\n"
     ]
    }
   ],
   "source": [
    "all_metrics = read_file_and_run_clustering_pipeline({\n",
    "    'log': '../process_mining_datasets\\\\fr\\\\fr5k.mxml',\n",
    "    'representation': 'binary',\n",
    "    'parameters': [{\n",
    "        'model': 'kmeans__k=3', \n",
    "        'sliding_window': False,\n",
    "        'window_size': 125,\n",
    "        'sliding_step': 1\n",
    "    }]\n",
    "}, return_result=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>Silhouette</th>\n",
       "      <th>DBi</th>\n",
       "      <th>centroids</th>\n",
       "      <th>avg_dist_between_centroids</th>\n",
       "      <th>std_dist_between_centroids</th>\n",
       "      <th>volume_list</th>\n",
       "      <th>radius_list</th>\n",
       "      <th>dist_intra_cluster_list</th>\n",
       "      <th>skewness_list</th>\n",
       "      <th>...</th>\n",
       "      <th>total_MSE</th>\n",
       "      <th>avg_MSE</th>\n",
       "      <th>count_non_zero_MSE</th>\n",
       "      <th>diff_avg_dist_between_centroids</th>\n",
       "      <th>diff_std_dist_between_centroids</th>\n",
       "      <th>diff_volume</th>\n",
       "      <th>diff_radius</th>\n",
       "      <th>diff_dist_intra_cluster</th>\n",
       "      <th>diff_skewness</th>\n",
       "      <th>diff_cluster_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>2</td>\n",
       "      <td>0.723323</td>\n",
       "      <td>0.697017</td>\n",
       "      <td>[[1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0,...</td>\n",
       "      <td>2.494916</td>\n",
       "      <td>0.346735</td>\n",
       "      <td>[56, 38, 31]</td>\n",
       "      <td>[1.2374368670764582, 1.4872495948362938, 1.354...</td>\n",
       "      <td>[0.3149839298012803, 0.9139844893551741, 1.052...</td>\n",
       "      <td>[0.26546593660094936, -0.5953856692612958, -0....</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>2</td>\n",
       "      <td>0.735449</td>\n",
       "      <td>0.677818</td>\n",
       "      <td>[[1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0,...</td>\n",
       "      <td>2.502506</td>\n",
       "      <td>0.340888</td>\n",
       "      <td>[62, 32, 31]</td>\n",
       "      <td>[1.2089244968673232, 1.4856080405005891, 1.460...</td>\n",
       "      <td>[0.3567318187477347, 0.9033037741280492, 1.009...</td>\n",
       "      <td>[0.25675899072182007, -0.5898857386498104, -0....</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018007</td>\n",
       "      <td>0.000948</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.00759</td>\n",
       "      <td>-0.005847</td>\n",
       "      <td>[6, -6, 0]</td>\n",
       "      <td>0.004051</td>\n",
       "      <td>0.001236</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>6.857610e-08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     k  Silhouette       DBi  \\\n",
       "i                              \n",
       "125  2    0.723323  0.697017   \n",
       "250  2    0.735449  0.677818   \n",
       "\n",
       "                                             centroids  \\\n",
       "i                                                        \n",
       "125  [[1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0,...   \n",
       "250  [[1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0,...   \n",
       "\n",
       "     avg_dist_between_centroids  std_dist_between_centroids   volume_list  \\\n",
       "i                                                                           \n",
       "125                    2.494916                    0.346735  [56, 38, 31]   \n",
       "250                    2.502506                    0.340888  [62, 32, 31]   \n",
       "\n",
       "                                           radius_list  \\\n",
       "i                                                        \n",
       "125  [1.2374368670764582, 1.4872495948362938, 1.354...   \n",
       "250  [1.2089244968673232, 1.4856080405005891, 1.460...   \n",
       "\n",
       "                               dist_intra_cluster_list  \\\n",
       "i                                                        \n",
       "125  [0.3149839298012803, 0.9139844893551741, 1.052...   \n",
       "250  [0.3567318187477347, 0.9033037741280492, 1.009...   \n",
       "\n",
       "                                         skewness_list  ... total_MSE  \\\n",
       "i                                                       ...             \n",
       "125  [0.26546593660094936, -0.5953856692612958, -0....  ...       NaN   \n",
       "250  [0.25675899072182007, -0.5898857386498104, -0....  ...  0.018007   \n",
       "\n",
       "      avg_MSE  count_non_zero_MSE  diff_avg_dist_between_centroids  \\\n",
       "i                                                                    \n",
       "125       NaN                 NaN                              NaN   \n",
       "250  0.000948                 4.0                          0.00759   \n",
       "\n",
       "     diff_std_dist_between_centroids  diff_volume  diff_radius  \\\n",
       "i                                                                \n",
       "125                              NaN          NaN          NaN   \n",
       "250                        -0.005847   [6, -6, 0]     0.004051   \n",
       "\n",
       "     diff_dist_intra_cluster  diff_skewness  diff_cluster_std  \n",
       "i                                                              \n",
       "125                      NaN            NaN               NaN  \n",
       "250                 0.001236       0.000035      6.857610e-08  \n",
       "\n",
       "[2 rows x 34 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_metrics.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Experiments with several parameters combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 18, 1350)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Trace clustering parameters\n",
    "grid_parameters = list(ParameterGrid({\n",
    "    \"sliding_window\": [False],    \n",
    "    \"window_size\": [75, 100, 125, 150, 175, 200],\n",
    "    'sliding_step': [1],\n",
    "    \"model\": [\n",
    "        'kmeans__k=6',\n",
    "        'kmeans__k=3',\n",
    "        'kmeans__k=2'\n",
    "    ] \n",
    "}))\n",
    "\n",
    "# Trace vector representations\n",
    "grid_logs = list(ParameterGrid([\n",
    "    {\n",
    "        \"log\": [x for x in logs if if_any(x, binary_support)],\n",
    "        \"representation\": ['binary']\n",
    "    },\n",
    "    {\n",
    "        \"log\": [x for x in logs if if_any(x, frequency_support)],\n",
    "        \"representation\": ['frequency']\n",
    "    },\n",
    "    {\n",
    "        \"log\": [x for x in logs if if_any(x, transitions_support)],\n",
    "        \"representation\": ['transitions']\n",
    "    }\n",
    "]))\n",
    "\n",
    "# Combining all parameters\n",
    "combs = []\n",
    "for x in grid_logs:\n",
    "    dic = x.copy()\n",
    "    dic['parameters'] = grid_parameters\n",
    "    \n",
    "    combs.append(dic)\n",
    "\n",
    "len(combs), len(grid_parameters), len(combs) * len(grid_parameters) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run parallely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1cc7db693d948639c9887d51f091c76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=75.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_resp = Parallel(n_jobs=3)(\n",
    "    delayed(read_file_and_run_clustering_pipeline)(comb) for comb in tqdm_notebook(combs)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detection Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drift detection parameters\n",
    "drift_config = list(ParameterGrid([\n",
    "    {\n",
    "        \"rolling_window\": [3, 4, 5],\n",
    "        \"std_tolerance\": [1.25, 1.5, 1.75, 2],\n",
    "        'min_tol': [0.01, 0.007, 0.005, 0.003] \n",
    "    }\n",
    "]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List all files obtained after the trace clustering pipeline\n",
    "\n",
    "clusterizacoes = glob.glob(OUTPUT_PATH + \"*/*.pickle.gzip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combinations to run\n",
    "\n",
    "combs_deteccao = []\n",
    "for log in clusterizacoes:\n",
    "    combs_deteccao.append({\n",
    "        'input': log,\n",
    "        'combinations': drift_config\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace part of the OUTPUT_PATH to create a new folder \n",
    "# for the detection results\n",
    "\n",
    "OLD_prefix = \"LoanApplications_Offline\"\n",
    "NEW_prefix = \"LoanApplications_Offline__DETECTION\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drift_detect_pipeline(args, return_results=False):\n",
    "    \"\"\"\n",
    "        Runs the drift detection method based on the output from the trace\n",
    "        clustering pipeline for different combination of parameters and every\n",
    "        feature from tracking the trace clustering evolution. The\n",
    "        outputs are into a new folder named by the NEW_prefix variable in \n",
    "        gzip csv files.\n",
    "        \n",
    "        Parameters:\n",
    "        -----------\n",
    "            args (dict): Dictionary with the parameters and the log_file path\n",
    "                requiring the following keys:\n",
    "                    example = {\n",
    "                    }\n",
    "    \"\"\"\n",
    "    # Read file\n",
    "    all_metrics = pd.read_pickle(args[\"input\"], compression='gzip')\n",
    "    \n",
    "    # Parse information from file name\n",
    "    path_file = args[\"input\"].replace(\".pickle.gzip\", \"\").split('\\\\')\n",
    "    args.update({\n",
    "        \"tipo_mudanca\": path_file[-1].split('_')[0],\n",
    "        \"log_size\": int(path_file[-1].split('_')[1]),\n",
    "        \"model\": \"_\".join(path_file[-1].split('_')[2:5]),\n",
    "        \"representation\": \"_\".join(path_file[-1].split('_')[5:-2]),\n",
    "        \"window_size\": path_file[-1].split('_')[-2],\n",
    "        \"sliding_window\": path_file[-1].split('_')[-1]\n",
    "    })\n",
    "    \n",
    "    # Run detection for every combination of parameter    \n",
    "    for combination in args['combinations']:\n",
    "        c = deepcopy(combination)\n",
    "        c.update({\n",
    "            'input': args['input'],\n",
    "            'tipo_mudanca': args['tipo_mudanca'],\n",
    "            'log_size': args['log_size'],\n",
    "            'model': args['model'],\n",
    "            'representation': args['representation'],\n",
    "            'window_size': args['window_size'],\n",
    "            'sliding_window': args['sliding_window']\n",
    "        })\n",
    "        \n",
    "        if return_results:\n",
    "            return __drift_detect_pipeline(\n",
    "                all_metrics, c, return_results\n",
    "            )\n",
    "        else:\n",
    "            __drift_detect_pipeline(\n",
    "                all_metrics, c, return_results\n",
    "            )\n",
    "    \n",
    "\n",
    "def __drift_detect_pipeline(all_metrics, args, return_results=False):     \n",
    "    base_name = args[\"input\"].replace(\".pickle.gzip\", \"\")\n",
    "    base_name = base_name.replace(OLD_prefix, NEW_prefix)\n",
    "    \n",
    "    # Create string with parameters to identify file\n",
    "    to_string = [\n",
    "        str(args[\"rolling_window\"]),\n",
    "        str(args[\"std_tolerance\"]).replace(\".\", \"-\"), \n",
    "        str(args[\"min_tol\"]).replace(\".\", \"-\")\n",
    "    ]\n",
    "    \n",
    "    try:\n",
    "        os.makedirs(base_name)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    final_name = base_name.replace(\"\\\\\", \"/\") + \"/\" + \"_\".join(to_string) + \".pickle.gzip\"\n",
    "    \n",
    "    if os.path.isfile(final_name):\n",
    "        if return_results:\n",
    "            return pd.read_pickle(final_name, compression='gzip')\n",
    "        else:\n",
    "            print(\"Already exists\")\n",
    "            return\n",
    "    \n",
    "    y_true = [x for x in range(int(args['log_size']/10), args['log_size'], int(args['log_size']/10))]\n",
    "    \n",
    "    # Runs the drift detection for every feature\n",
    "    results = []\n",
    "    for col in all_metrics.select_dtypes(include=np.number).columns:\n",
    "        if (col not in [\"k\"] and not col.startswith(\"diff\") ) or col in [\"diff_centroids\"]:\n",
    "            r = deepcopy(args)\n",
    "            r[\"measure\"] = col\n",
    "\n",
    "            detected_drifts, extra = dd.detect_concept_drift(\n",
    "                all_metrics, \n",
    "                col,\n",
    "                args[\"rolling_window\"],\n",
    "                args[\"std_tolerance\"],\n",
    "                args[\"min_tol\"]\n",
    "            )    \n",
    "    \n",
    "            # Calculate classification metrics\n",
    "            metrics_results = dd.get_metrics(\n",
    "                detected_drifts,\n",
    "                y_true,\n",
    "                int(args[\"window_size\"])\n",
    "            )\n",
    "\n",
    "            r.update(args)\n",
    "            r.update(metrics_results)\n",
    "\n",
    "            results.append(r)\n",
    "\n",
    "            gc.collect()\n",
    "    \n",
    "    # Export as file\n",
    "    pd.DataFrame(results).to_pickle(\n",
    "        final_name,\n",
    "        compression=\"gzip\"\n",
    "    )\n",
    "    \n",
    "    if return_results:\n",
    "        return pd.DataFrame(results)\n",
    "    \n",
    "    # print(col, len(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "detection_results = drift_detect_pipeline({\n",
    "    'input': '../LoanApplications_Offline\\\\cb\\\\cb_10000_kmeans__k=2_binary_100_False.pickle.gzip',\n",
    "    'combinations': [{\n",
    "       'min_tol': 0.01,\n",
    "       'rolling_window': 3,\n",
    "       'std_tolerance': 1.25\n",
    "    }, {\n",
    "       'min_tol': 0.02,\n",
    "       'rolling_window': 3,\n",
    "       'std_tolerance': 1.25\n",
    "    }]\n",
    "}, return_results=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>min_tol</th>\n",
       "      <th>rolling_window</th>\n",
       "      <th>std_tolerance</th>\n",
       "      <th>input</th>\n",
       "      <th>tipo_mudanca</th>\n",
       "      <th>log_size</th>\n",
       "      <th>model</th>\n",
       "      <th>representation</th>\n",
       "      <th>window_size</th>\n",
       "      <th>sliding_window</th>\n",
       "      <th>measure</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Delay</th>\n",
       "      <th>Correct_Predictions</th>\n",
       "      <th>Support</th>\n",
       "      <th>Drifts_Found</th>\n",
       "      <th>Resp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.01</td>\n",
       "      <td>3</td>\n",
       "      <td>1.25</td>\n",
       "      <td>../LoanApplications_Offline\\cb\\cb_10000_kmeans...</td>\n",
       "      <td>cb</td>\n",
       "      <td>10000</td>\n",
       "      <td>kmeans__k=2</td>\n",
       "      <td>binary</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>avg_cluster_std</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>[1100, 2100, 3100, 4100, 5100, 6100, 7100, 810...</td>\n",
       "      <td>[1000, 2000, 3000, 4000, 5000, 6000, 7000, 800...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.01</td>\n",
       "      <td>3</td>\n",
       "      <td>1.25</td>\n",
       "      <td>../LoanApplications_Offline\\cb\\cb_10000_kmeans...</td>\n",
       "      <td>cb</td>\n",
       "      <td>10000</td>\n",
       "      <td>kmeans__k=2</td>\n",
       "      <td>binary</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>count_non_zero_MSE</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>150.0</td>\n",
       "      <td>[1, 1, 1, 1, 0, 1, 1, 1, 1]</td>\n",
       "      <td>8</td>\n",
       "      <td>[1100, 2200, 3100, 4200, 4900, 5500, 6200, 710...</td>\n",
       "      <td>[1000, 2000, 3000, 4000, 5000, 6000, 7000, 800...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    min_tol  rolling_window  std_tolerance  \\\n",
       "10     0.01               3           1.25   \n",
       "16     0.01               3           1.25   \n",
       "\n",
       "                                                input tipo_mudanca  log_size  \\\n",
       "10  ../LoanApplications_Offline\\cb\\cb_10000_kmeans...           cb     10000   \n",
       "16  ../LoanApplications_Offline\\cb\\cb_10000_kmeans...           cb     10000   \n",
       "\n",
       "          model representation window_size sliding_window             measure  \\\n",
       "10  kmeans__k=2         binary         100          False     avg_cluster_std   \n",
       "16  kmeans__k=2         binary         100          False  count_non_zero_MSE   \n",
       "\n",
       "    Precision    Recall        F1  Delay          Correct_Predictions  \\\n",
       "10        1.0  1.000000  1.000000  100.0  [1, 1, 1, 1, 1, 1, 1, 1, 1]   \n",
       "16        0.8  0.888889  0.842105  150.0  [1, 1, 1, 1, 0, 1, 1, 1, 1]   \n",
       "\n",
       "    Support                                       Drifts_Found  \\\n",
       "10        9  [1100, 2100, 3100, 4100, 5100, 6100, 7100, 810...   \n",
       "16        8  [1100, 2200, 3100, 4200, 4900, 5500, 6200, 710...   \n",
       "\n",
       "                                                 Resp  \n",
       "10  [1000, 2000, 3000, 4000, 5000, 6000, 7000, 800...  \n",
       "16  [1000, 2000, 3000, 4000, 5000, 6000, 7000, 800...  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detection_results.sort_values('F1', ascending=False).head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run parallely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14a21195ec844156b2a711a25010295e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=54.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_resp = Parallel(n_jobs=3)(\n",
    "    delayed(drift_detect_pipeline)(comb_d) for comb_d in tqdm_notebook(combs_deteccao)\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}