testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import stumpy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test_data(data_folder: str, selected_folder: str, label_filename: str):\n",
    "    files = [f.split(\".\")[0] for f in os.listdir(f\"{data_folder}/{selected_folder}\")]\n",
    "    labels = pd.read_csv(\n",
    "        f\"{data_folder}/{selected_folder}/{label_filename}.csv\", index_col=0\n",
    "    )\n",
    "    train_files = [file for file in files if file != label_filename]\n",
    "\n",
    "    return train_files, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def from_files_to_anomaly_type(train_files: list) -> dict:\n",
    "    from_files_to_anomaly_type = {}\n",
    "\n",
    "    for file in train_files:\n",
    "        if file.startswith(\"1\"):\n",
    "            from_files_to_anomaly_type[file] = \"bursty input\"\n",
    "        elif file.startswith(\"2\"):\n",
    "            from_files_to_anomaly_type[file] = \"stalled input\"\n",
    "        elif file.startswith(\"3\"):\n",
    "            from_files_to_anomaly_type[file] = \"CPU contention\"\n",
    "        else:\n",
    "            raise ValueError(f\"Unknown file {file}.\")\n",
    "\n",
    "    return from_files_to_anomaly_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_anomalies_and_references(\n",
    "    train_files: list,\n",
    "    labels: pd.DataFrame,\n",
    "    data_folder: str,\n",
    "    selected_folder: str,\n",
    "    from_files_to_anomaly_type: dict,\n",
    ") -> (dict, dict):\n",
    "    references = {}\n",
    "    anomalies = {}\n",
    "\n",
    "    for filename in train_files:\n",
    "        train_file = pd.read_csv(\n",
    "            f\"{data_folder}/{selected_folder}/{filename}.csv\", index_col=0\n",
    "        )\n",
    "        train_file[\"original_filename\"] = filename\n",
    "        train_file[\"timestamp\"] = train_file.index\n",
    "\n",
    "        label_file = labels.loc[labels[\"trace_id\"] == filename, :]\n",
    "\n",
    "        for i in label_file.index:\n",
    "            ano_id = label_file.loc[i, \"ano_id\"]\n",
    "            selection_ref = train_file.loc[\n",
    "                (train_file[\"timestamp\"] >= label_file[\"ref_start\"][i])\n",
    "                & (train_file[\"timestamp\"] < label_file[\"ref_end\"][i]),\n",
    "                :,\n",
    "            ].copy()\n",
    "            selection_ref[\"ano_id\"] = ano_id\n",
    "            selection_ref[\"type_data\"] = 0\n",
    "            selection_ano = train_file.loc[\n",
    "                (train_file[\"timestamp\"] >= label_file[\"ano_start\"][i])\n",
    "                & (train_file[\"timestamp\"] <= label_file[\"ano_end\"][i]),\n",
    "                :,\n",
    "            ].copy()\n",
    "            selection_ano[\"ano_id\"] = ano_id\n",
    "            selection_ano[\"type_data\"] = 1\n",
    "            references[\n",
    "                f\"{from_files_to_anomaly_type[filename]}_{filename}_{i}\"\n",
    "            ] = selection_ref\n",
    "            anomalies[\n",
    "                f\"{from_files_to_anomaly_type[filename]}_{filename}_{i}\"\n",
    "            ] = selection_ano\n",
    "\n",
    "    assert references.keys() == anomalies.keys()\n",
    "    references = pd.concat(references).droplevel(1)\n",
    "    anomalies = pd.concat(anomalies).droplevel(1)\n",
    "\n",
    "    return references, anomalies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_correlated_features(df, correlation_threshold=0.9):\n",
    "    # Step 1: Calculate the correlation matrix\n",
    "    correlation_matrix = df.corr()\n",
    "\n",
    "    # Step 2: Create a graph based on pairwise correlations\n",
    "    G = nx.Graph()\n",
    "\n",
    "    # Add nodes (features) to the graph\n",
    "    G.add_nodes_from(correlation_matrix.columns)\n",
    "\n",
    "    # Add edges between nodes if the correlation exceeds a threshold\n",
    "    for i in range(len(correlation_matrix.columns[:-4])):  # Last 4 columns are metadata\n",
    "        for j in range(i):\n",
    "            if abs(correlation_matrix.iloc[i, j]) > correlation_threshold:\n",
    "                G.add_edge(correlation_matrix.columns[i], correlation_matrix.columns[j])\n",
    "\n",
    "    # Step 3: Extract clusters from the graph\n",
    "    clusters = list(nx.connected_components(G))\n",
    "\n",
    "    # Step 4: Select one representative feature from each cluster\n",
    "    selected_features = [cluster.pop() for cluster in clusters]\n",
    "\n",
    "    return selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def false_positive_filtering(refs, anos, cols):\n",
    "    new_cols = []\n",
    "    refs_df = refs[cols]\n",
    "    anos_df = anos[cols]\n",
    "    cols_to_visit = list(anos_df.columns[:-4])\n",
    "    for ano in anos.index.unique():\n",
    "        cols_for_this_ano = []\n",
    "        nb_matches = []\n",
    "        for col in cols_to_visit:\n",
    "            pattern = anos_df.loc[ano, col]\n",
    "            ts = refs_df.loc[:, col]\n",
    "            matches = stumpy.match(pattern, ts, max_distance=28.0)\n",
    "            nb_matches.append(len(matches))\n",
    "            if len(list(matches)) <= 1:\n",
    "                if col not in new_cols:\n",
    "                    cols_for_this_ano.append(col)\n",
    "            # else:\n",
    "            # print(f\"Found {len(matches)} match(es) for {col} in ano {ano}\")\n",
    "        if not cols_for_this_ano:\n",
    "            new_cols.append(cols_to_visit[np.array(nb_matches).argmin()])\n",
    "        else:\n",
    "            new_cols.append(cols_for_this_ano)\n",
    "    return new_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def assign_cols_per_ano(anos, new_filtered_features):\n",
    "    anos[\"filtered_columns\"] = None\n",
    "    for i, ano in enumerate(anos.index.unique()):\n",
    "        anos.loc[ano, \"filtered_columns\"] = str(new_filtered_features[i])\n",
    "    anos[\"filtered_columns\"] = anos[\"filtered_columns\"].apply(\n",
    "        lambda x: x.strip(\"][\").split(\", \")\n",
    "    )\n",
    "    return anos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_entropy(nb_ts_a: list, nb_ts_r: list) -> float:\n",
    "    \"\"\"\n",
    "    Calculate the class entropy of a feature, which is the information\n",
    "    needed to describe the class distributions between two time serie.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    nb_ts_a : list\n",
    "        Number of observations inside a time series belonging to the abnormal\n",
    "        class.\n",
    "    nb_ts_r : list\n",
    "        Number of observations inside a time series belonging to the reference\n",
    "        class.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    float\n",
    "        The class entropy.\n",
    "    \"\"\"\n",
    "\n",
    "    if nb_ts_a == 0 or nb_ts_r == 0:\n",
    "        raise ValueError(\n",
    "            f\"One of the time series is empty. Len of TSA is {nb_ts_a} and len of TSR\"\n",
    "            f\" is {nb_ts_r}.\"\n",
    "        )\n",
    "    p_a = nb_ts_a / (nb_ts_a + nb_ts_r)\n",
    "    p_r = nb_ts_r / (nb_ts_a + nb_ts_r)\n",
    "    h_class = p_a * np.log2(1 / p_a) + p_r * np.log2(1 / p_r)\n",
    "\n",
    "    return h_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_observations_if_duplicates(\n",
    "    sorted_values: pd.DataFrame, feature\n",
    ") -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Shuffle the observations if there are duplicates in the sorted values.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    sorted_values : pd.DataFrame\n",
    "        The sorted values.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "        The sorted values with shuffled values for duplicates.\n",
    "    \"\"\"\n",
    "\n",
    "    # On récupère le nombre de références et d'anomalies pour chaque modalité\n",
    "    value_type_to_count = sorted_values.groupby(feature).value_counts().to_dict()\n",
    "\n",
    "    # On récupère le nombre de valeurs distinctes pour chaque modalité (soit 1 lorsque\n",
    "    # pas de doublons, soit 2, lorsqu'il y a des doublons)\n",
    "    value_to_count_distinct = (\n",
    "        sorted_values.drop_duplicates().groupby(feature).count().to_dict()[\"type_data\"]\n",
    "    )\n",
    "\n",
    "    # On récupère les modalités distinctes (les différentes valeurs prises par la\n",
    "    # feature)\n",
    "    modalities = set(sorted_values[feature].tolist())\n",
    "    # En fait, ce qui est un peu bizarre c'est qu'on a des valeurs continues, mais là\n",
    "    # on va les traiter comme des valeurs discrètes :\n",
    "    # Par exemple si on considère une colonne qui prend les valeurs 501.03, 501.03,\n",
    "    # 502.4, 502.4, 505.0, on itère sur 501.03, 502.4, 505.0\n",
    "\n",
    "    # On parcourt chaque modalité\n",
    "    for modality in modalities:\n",
    "        # On récupère le premier type de données observé (ano ou ref, donc 1 ou 0) (ça\n",
    "        # nous sera utile plus tard)\n",
    "        last_type_data = sorted_values.loc[\n",
    "            sorted_values[feature] == modality, \"type_data\"\n",
    "        ].tolist()[0]\n",
    "\n",
    "        # Cas où il n'y a pas de doublons\n",
    "        if value_to_count_distinct[modality] == 1:\n",
    "            # On ne fait rien\n",
    "            continue\n",
    "\n",
    "        # Cas où il y a des doublons\n",
    "        else:\n",
    "            # On va shuffle dans le pire ordre possible\n",
    "\n",
    "            # D'abord on instancie les variables nécessaires\n",
    "            list_values = []\n",
    "            nb_refs = value_type_to_count[(modality, 0)]\n",
    "            nb_anos = value_type_to_count[(modality, 1)]\n",
    "            nb_total = nb_refs + nb_anos\n",
    "            # Hop maintenant c'est parti pour le shuffle\n",
    "\n",
    "            # Cas où il n'y a pas le même nombre de références et d'anomalie\n",
    "            #  (cas le plus chiant)\n",
    "            if nb_refs != nb_anos:\n",
    "                # On instancie de nouveau des variables nécessaires\n",
    "                biggest = int(\n",
    "                    nb_refs < nb_anos\n",
    "                )  # 1 si on a plus d'anomalies que de références, 0 sinon\n",
    "                smallest = int(\n",
    "                    nb_refs > nb_anos\n",
    "                )  # 1 si on a plus de références que d'anomalies, 0 sinon\n",
    "                nb_smallest = min(\n",
    "                    nb_refs, nb_anos\n",
    "                )  # Nombre de fois où on va mettre la valeur la moins représentée\n",
    "\n",
    "                # On commence par mettre la valeur la plus représentée partout\n",
    "                list_values = [biggest] * nb_total\n",
    "\n",
    "                # Puis on cherche si le dernier type de donnée observé est le plus\n",
    "                # représenté ou le moins représenté pour savoir où commencer\n",
    "                start_smallest = 0 if smallest != last_type_data else 1\n",
    "\n",
    "                # On parcourt la liste 2 par 2 pour mettre la valeur la moins\n",
    "                # représentée\n",
    "                for i in range(start_smallest, nb_smallest * 2, 2):\n",
    "                    list_values[i] = smallest\n",
    "\n",
    "            # Cas où il y a le même nombre de références et d'anomalies\n",
    "            # (cas le plus simple)\n",
    "            else:\n",
    "                # On parcourt le nombre total d'observations\n",
    "                for i in range(nb_total):\n",
    "                    # On alterne entre 0 et 1 en commençant par la valeur opposée à la\n",
    "                    # dernière valeur observée\n",
    "                    list_values.append(abs(last_type_data - i % 2 - 1))\n",
    "\n",
    "            # On récupère le dernier type de donnée observé (toujours 1 ou 0)\n",
    "            last_type_data = sorted_values.loc[\n",
    "                sorted_values[feature] == modality, \"type_data\"\n",
    "            ].tolist()[-1]\n",
    "\n",
    "        # On vérifie que la longueur de la liste est bien égale au nombre\n",
    "        # d'observations pour la modalité\n",
    "        assert (\n",
    "            len(list_values)\n",
    "            == sorted_values[sorted_values[feature] == modality].shape[0]\n",
    "        ), (\n",
    "            f\"Len of list_values {len(list_values)} is not equal to the number of\"\n",
    "            \" observations for the modality\"\n",
    "            f\" {sorted_values[sorted_values[feature]==modality].shape[0]}.\"\n",
    "        )\n",
    "        # On met à jour le type de donnée pour la modalité\n",
    "        sorted_values.loc[sorted_values[feature] == modality, \"type_data\"] = list_values\n",
    "\n",
    "    return sorted_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def segmentation_entropy(shuffled_values: pd.DataFrame) -> float:\n",
    "    \"\"\"\n",
    "    Calculate the segmentation entropy of a feature, which is the information\n",
    "    needed to describe how merged points are segmented by class labels.\n",
    "\n",
    "    Parameters:\n",
    "    shuffled_values (pd.DataFrame): A DataFrame containing the shuffled values.\n",
    "\n",
    "    Returns:\n",
    "    float: The segmentation entropy of the feature.\n",
    "    \"\"\"\n",
    "    # On récupère la time serie\n",
    "    ts = shuffled_values[\"type_data\"].tolist()\n",
    "\n",
    "    # Stocke la première valeur de la liste\n",
    "    past_value = ts[0]\n",
    "\n",
    "    # Liste pour stocker les valeurs à l'intérieur d'un segment\n",
    "    values_inside_segment = []\n",
    "\n",
    "    # Variable pour stocker la segmentation entropy\n",
    "    segmentation_ent = 0.0\n",
    "\n",
    "    # Parcourt chaque valeur dans la time serie\n",
    "    for value in ts:\n",
    "        # Si la valeur est différente de la valeur précédente\n",
    "        if value != past_value:\n",
    "            # On a un nouveau segment, il faut calculer l'entropie de segmentation\n",
    "            # partielle du précédent segment\n",
    "            pi = len(values_inside_segment) / shuffled_values.shape[0]\n",
    "            segmentation_ent += pi * np.log(1 / pi)\n",
    "\n",
    "            # On réinitialise la liste des valeurs à l'intérieur du segment avec la\n",
    "            # nouvelle valeur\n",
    "            values_inside_segment = [value]\n",
    "        else:\n",
    "            # On stocke les valeurs à l'intérieur du segment tant qu'un nouveau segment\n",
    "            # n'est pas créé\n",
    "            values_inside_segment.append(value)\n",
    "\n",
    "        # On met à jour la valeur précédente avec la valeur actuelle\n",
    "        past_value = value\n",
    "\n",
    "    return segmentation_ent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entropy_based_single_feature_reward(refs: pd.DataFrame, anos: pd.DataFrame) -> dict:\n",
    "    \"\"\"\n",
    "    Calculates the reward function for a single feature based on the reference\n",
    "    data and the annotated data.\n",
    "\n",
    "    Parameters:\n",
    "    refs (pandas.DataFrame): The reference data.\n",
    "    anos (pandas.DataFrame): The abnormal data.\n",
    "\n",
    "    Returns:\n",
    "    dict: A dictionary containing the single reward function for each feature.\n",
    "    \"\"\"\n",
    "\n",
    "    distances = {}\n",
    "    # On calcule la class entropy\n",
    "    class_ent = class_entropy(refs.shape[0], anos.shape[0])\n",
    "    # On calcule la segmentation entropy pour chaque feature sauf type_data\n",
    "    for feature in [col for col in refs.columns if col != \"type_data\"]:\n",
    "        # On concatène les références et les anomalies pour la feature\n",
    "        all_values = pd.concat(\n",
    "            [refs[[feature, \"type_data\"]], anos[[feature, \"type_data\"]]]\n",
    "        )\n",
    "        # On trie les valeurs par feature puis par type_data\n",
    "        sorted_values = all_values.sort_values(by=[feature, \"type_data\"])\n",
    "        # On shuffle les valeurs si on a des doublons\n",
    "        shuffled_values = shuffle_observations_if_duplicates(sorted_values, feature)\n",
    "        # On calcule la segmentation entropy\n",
    "        segmentation_ent = segmentation_entropy(shuffled_values)\n",
    "        # On calcule la single reward function\n",
    "        distance = class_ent / segmentation_ent\n",
    "        # On stocke la single reward function dans le dictionnaire\n",
    "        distances[feature] = distance\n",
    "\n",
    "        # if all(value < 0 for value in distances.values()):\n",
    "        #    positive_distances = {\n",
    "        #        key: np.abs(value) for key, value in distances.items()\n",
    "        #    }\n",
    "\n",
    "        sorted_distances = dict(\n",
    "            sorted(distances.items(), key=lambda x: x[1], reverse=True)\n",
    "        )\n",
    "\n",
    "    return sorted_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maximum_leap(distances: dict) -> float:\n",
    "    \"\"\"\n",
    "    Calculate the maximum leap between the distances of the neighboring ranked\n",
    "    features.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    distances : dict\n",
    "        The ranking of all remaining features based on their individual reward.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    maximum_leap : float\n",
    "        The maximum leap.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    leaps = [\n",
    "        last_distance - distance\n",
    "        for last_distance, distance in zip(\n",
    "            distances.values(), list(distances.values())[1:]\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    maximum_leap = max(leaps)\n",
    "\n",
    "    return maximum_leap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reward_leap_filter(distances: dict) -> list:\n",
    "    \"\"\"\n",
    "    Discard the features that rank below a sharp drop in the reward.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    distances : dict\n",
    "        The ranking of all remaining features based on their individual reward.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    filtered_features : dict\n",
    "        The remaining features after filtering.\n",
    "\n",
    "    \"\"\"\n",
    "    if len(distances) > 0:  # j'ai ajouté ça au cas où\n",
    "        threshold = maximum_leap(distances)\n",
    "        to_keep = set()\n",
    "        last_distance = 0\n",
    "\n",
    "        for feature, distance in distances.items():\n",
    "            if last_distance != 0:\n",
    "                leap = last_distance - distance\n",
    "                if leap == threshold:\n",
    "                    break\n",
    "            last_distance = distance\n",
    "            to_keep.update([feature])\n",
    "\n",
    "        filtered_features = [\n",
    "            feature for feature in distances.keys() if feature in to_keep\n",
    "        ]\n",
    "\n",
    "        return filtered_features\n",
    "\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = \"data\"\n",
    "FOLDERS = [\"folder_1\", \"folder_2\"]\n",
    "SEL_FOLDER = FOLDERS[0]\n",
    "train_files, labels = get_train_test_data(DATA_FOLDER, SEL_FOLDER, \"labels\")\n",
    "from_files_to_anomaly_type = from_files_to_anomaly_type(train_files)\n",
    "refs, anos = split_anomalies_and_references(\n",
    "    train_files, labels, DATA_FOLDER, SEL_FOLDER, from_files_to_anomaly_type\n",
    ")\n",
    "all_data = pd.concat([refs, anos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 1\n",
    "filtered_features = remove_correlated_features(all_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP2\n",
    "new_filtered_features = false_positive_filtering(refs, anos, filtered_features)\n",
    "new_anos = assign_cols_per_ano(anos, new_filtered_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_anos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# distances = []\n",
    "\n",
    "# for ano_index in new_anos.index.unique():\n",
    "#     ano = new_anos.loc[ano_index]\n",
    "#     columns_list = ano[\"filtered_columns\"].values\n",
    "#     print(ano_index)\n",
    "#     for columns in np.unique(columns_list):\n",
    "#         columns = [s.replace(\"'\", \"\") for s in columns]\n",
    "#         filtered_ano = ano[columns + [\"type_data\"]]\n",
    "#         ref = refs[columns + [\"type_data\"]]\n",
    "#         print(filtered_ano.shape)\n",
    "#         distance = entropy_based_single_feature_reward(ref, filtered_ano)\n",
    "#         print(distance)\n",
    "#         distances.append(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bursty_refs = refs[refs.index.str.startswith(\"bursty\")]\n",
    "bursty_anos = new_anos[anos.index.str.startswith(\"bursty\")]\n",
    "stalled_refs = refs[refs.index.str.startswith(\"stalled\")]\n",
    "stalled_anos = new_anos[anos.index.str.startswith(\"stalled\")]\n",
    "cpu_refs = refs[refs.index.str.startswith(\"CPU\")]\n",
    "cpu_anos = new_anos[anos.index.str.startswith(\"CPU\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 3\n",
    "\n",
    "distances_bursty = entropy_based_single_feature_reward(\n",
    "    bursty_refs,\n",
    "    bursty_anos,\n",
    ")\n",
    "\n",
    "distances_stalled = entropy_based_single_feature_reward(\n",
    "    stalled_refs,\n",
    "    stalled_anos,\n",
    ")\n",
    "\n",
    "distances_cpu = entropy_based_single_feature_reward(\n",
    "    cpu_refs,\n",
    "    cpu_anos,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances_bursty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances_stalled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances_cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_explanatory_features(anos: pd.DataFrame, distances: dict) -> dict:\n",
    "    selected_features = {}\n",
    "\n",
    "    for ano_index in anos.index.unique():\n",
    "        ano = anos.loc[ano_index]\n",
    "        filtered_cols = ano[\"filtered_columns\"].values\n",
    "        for cols in np.unique(filtered_cols):\n",
    "            cols = [s.replace(\"'\", \"\") for s in cols]\n",
    "            selected_distances = {\n",
    "                feat: dist for feat, dist in distances.items() if feat in cols\n",
    "            }\n",
    "            if len(selected_distances) > 1:\n",
    "                filtered_features = reward_leap_filter(selected_distances)\n",
    "                print(filtered_features)\n",
    "                selected_features[ano_index] = filtered_features\n",
    "            else:\n",
    "                selected_features[ano_index] = list(selected_distances.keys())\n",
    "\n",
    "    return selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features_bursty = get_explanatory_features(bursty_anos, distances_bursty)\n",
    "selected_features_bursty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features_stalled = get_explanatory_features(stalled_anos, distances_stalled)\n",
    "selected_features_stalled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features_cpu = get_explanatory_features(cpu_anos, distances_cpu)\n",
    "selected_features_cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ano_index in stalled_anos.index.unique():\n",
    "    stalled_ano = stalled_anos.loc[ano_index]\n",
    "    columns_list = stalled_ano[\"filtered_columns\"].values\n",
    "    print(ano_index)\n",
    "    for i, columns in enumerate(np.unique(columns_list)):\n",
    "        fig, ax = plt.subplots(1, 1)\n",
    "        columns = [s.replace(\"'\", \"\") for s in columns]\n",
    "        selected_distances_stalled = {\n",
    "            feat: dist for feat, dist in distances_stalled.items() if feat in columns\n",
    "        }\n",
    "        print(selected_distances_stalled)\n",
    "        print(maximum_leap(selected_distances_stalled))\n",
    "        ax.bar(selected_distances_stalled.keys(), selected_distances_stalled.values())\n",
    "        ax.axhline(y=np.median(list(selected_distances_stalled.values())), c=\"r\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ano_index in bursty_anos.index.unique():\n",
    "    bursty_ano = bursty_anos.loc[ano_index]\n",
    "    columns_list = bursty_ano[\"filtered_columns\"].values\n",
    "    # print(ano_index)\n",
    "    for i, columns in enumerate(np.unique(columns_list)):\n",
    "        columns = [s.replace(\"'\", \"\") for s in columns]\n",
    "        selected_distances_bursty = {\n",
    "            feat: dist for feat, dist in distances_bursty.items() if feat in columns\n",
    "        }\n",
    "        # print(selected_distances_bursty)\n",
    "        selected_features_bursty = reward_leap_filter(selected_distances_bursty)\n",
    "        # print(selected_features_bursty)\n",
    "\n",
    "        fig, ax = plt.subplots(1, 1)\n",
    "\n",
    "        features_names = list(selected_distances_bursty.keys())\n",
    "\n",
    "        all_distances = list(selected_distances_bursty.values())\n",
    "\n",
    "        colors = [\n",
    "            \"orange\" if feature in selected_features_bursty else \"b\"\n",
    "            for feature in features_names\n",
    "        ]\n",
    "\n",
    "        features_names = [\n",
    "            \"...\" + feature[-20:] if len(feature) > 10 else feature\n",
    "            for feature in features_names\n",
    "        ]\n",
    "\n",
    "        ax.bar(features_names, all_distances, color=colors)\n",
    "        ax.set_xticklabels(features_names, rotation=90)\n",
    "        ax.set_title(f\"Anomaly {ano_index}\")\n",
    "        # ax.axhline(y=np.median(list(selected_distances_bursty.values())), c=\"r\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ano_index in cpu_anos.index.unique():\n",
    "    cpu_ano = cpu_anos.loc[ano_index]\n",
    "    columns_list = cpu_ano[\"filtered_columns\"].values\n",
    "    # print(ano_index)\n",
    "    for i, columns in enumerate(np.unique(columns_list)):\n",
    "        try:\n",
    "            columns = [s.replace(\"'\", \"\") for s in columns]\n",
    "            selected_distances_cpu = {\n",
    "                feat: dist for feat, dist in distances_cpu.items() if feat in columns\n",
    "            }\n",
    "            # print(selected_distances_cpu)\n",
    "            selected_features_cpu = reward_leap_filter(selected_distances_cpu)\n",
    "            # print(selected_features_cpu)\n",
    "\n",
    "            fig, ax = plt.subplots(1, 1)\n",
    "\n",
    "            features_names = list(selected_distances_cpu.keys())\n",
    "\n",
    "            all_distances = list(selected_distances_cpu.values())\n",
    "\n",
    "            colors = [\n",
    "                \"orange\" if feature in selected_features_cpu else \"b\"\n",
    "                for feature in features_names\n",
    "            ]\n",
    "\n",
    "            features_names = [\n",
    "                \"...\" + feature[-20:] if len(feature) > 10 else feature\n",
    "                for feature in features_names\n",
    "            ]\n",
    "\n",
    "            ax.bar(features_names, all_distances, color=colors)\n",
    "            ax.set_xticklabels(features_names, rotation=90)\n",
    "            ax.set_title(f\"Anomaly {ano_index}\")\n",
    "            # ax.axhline(y=np.median(list(selected_distances_cpu.values())), c=\"r\")\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ano_index in stalled_anos.index.unique():\n",
    "    stalled_ano = stalled_anos.loc[ano_index]\n",
    "    columns_list = stalled_ano[\"filtered_columns\"].values\n",
    "    # print(ano_index)\n",
    "    for i, columns in enumerate(np.unique(columns_list)):\n",
    "        columns = [s.replace(\"'\", \"\") for s in columns]\n",
    "        selected_distances_stalled = {\n",
    "            feat: dist for feat, dist in distances_stalled.items() if feat in columns\n",
    "        }\n",
    "        # print(selected_distances_stalled)\n",
    "        selected_features_stalled = reward_leap_filter(selected_distances_stalled)\n",
    "        # print(selected_features_stalled)\n",
    "\n",
    "        fig, ax = plt.subplots(1, 1)\n",
    "\n",
    "        features_names = list(selected_distances_stalled.keys())\n",
    "\n",
    "        all_distances = list(selected_distances_stalled.values())\n",
    "\n",
    "        colors = [\n",
    "            \"orange\" if feature in selected_features_stalled else \"b\"\n",
    "            for feature in features_names\n",
    "        ]\n",
    "\n",
    "        features_names = [\n",
    "            \"...\" + feature[-20:] if len(feature) > 10 else feature\n",
    "            for feature in features_names\n",
    "        ]\n",
    "\n",
    "        ax.bar(features_names, all_distances, color=colors)\n",
    "        ax.set_xticklabels(features_names, rotation=90)\n",
    "        ax.set_title(f\"Anomaly {ano_index}\")\n",
    "        # ax.axhline(y=np.median(list(selected_distances_stalled.values())), c=\"r\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features_bursty, selected_features_stalled, selected_features_cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanatory_features = {\n",
    "    **selected_features_bursty,\n",
    "    **selected_features_stalled,\n",
    "    **selected_features_cpu,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(explanatory_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanatory_features_df = pd.DataFrame(\n",
    "    list(explanatory_features.items()), columns=[\"index\", \"explanation\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanatory_features_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_df = labels[[\"trace_id\", \"ano_id\"]].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations = pd.merge(\n",
    "    labels_df, explanatory_features_df, left_index=True, right_index=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations.drop([\"index\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_features_integer_indice(features: list, anomalies: pd.DataFrame):\n",
    "    \"\"\"\n",
    "    Returns the indices of the specified features in the anomalies DataFrame.\n",
    "\n",
    "    Parameters:\n",
    "    - features (list): A list of feature names.\n",
    "    - anomalies (pd.DataFrame): A DataFrame containing the anomalies.\n",
    "\n",
    "    Returns:\n",
    "    - indices (list): A list of integer indices corresponding to the features in the\n",
    "    anomalies DataFrame.\n",
    "    \"\"\"\n",
    "    indices = []\n",
    "    for feature in features:\n",
    "        indice = anomalies.columns.get_loc(feature)\n",
    "        indices.append(indice)\n",
    "\n",
    "    return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations[\"explanation\"] = explanations[\"explanation\"].apply(\n",
    "    lambda x: get_features_integer_indice(x, anos)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations[\"exp_size\"] = explanations[\"explanation\"].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations = [\n",
    "    [\n",
    "        \"driver_StreamingMetrics_streaming_lastCompletedBatch_schedulingDelay_value\",\n",
    "        \"driver_StreamingMetrics_streaming_lastReceivedBatch_records_value\",\n",
    "    ],\n",
    "    [\n",
    "        \"driver_StreamingMetrics_streaming_lastCompletedBatch_schedulingDelay_value\",\n",
    "        \"driver_StreamingMetrics_streaming_lastReceivedBatch_records_value\",\n",
    "    ],\n",
    "    [\n",
    "        \"driver_StreamingMetrics_streaming_lastCompletedBatch_schedulingDelay_value\",\n",
    "        \"driver_StreamingMetrics_streaming_lastReceivedBatch_records_value\",\n",
    "    ],\n",
    "    [\n",
    "        \"driver_StreamingMetrics_streaming_lastCompletedBatch_schedulingDelay_value\",\n",
    "        \"driver_StreamingMetrics_streaming_lastReceivedBatch_records_value\",\n",
    "    ],\n",
    "    [\n",
    "        \"driver_StreamingMetrics_streaming_lastCompletedBatch_schedulingDelay_value\",\n",
    "        \"driver_StreamingMetrics_streaming_lastReceivedBatch_records_value\",\n",
    "    ],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_instability(explanations: list):\n",
    "    \"\"\"\n",
    "    Compute the instability of a list of explanations.\n",
    "\n",
    "    Parameters:\n",
    "    explanations (list): A list of explanations.\n",
    "\n",
    "    Returns:\n",
    "    float: The instability value, which is a measure of how stable the explanations are.\n",
    "    \"\"\"\n",
    "    instability = 0\n",
    "    flattened_explanations = [item for sublist in explanations for item in sublist]\n",
    "    unique_explanations = set(flattened_explanations)\n",
    "    for feature in unique_explanations:\n",
    "        p = flattened_explanations.count(feature) / len(flattened_explanations)\n",
    "        instability += -p * np.log2(p)\n",
    "    # instability = 1 - len(unique_explanations) / len(explanations)\n",
    "\n",
    "    return instability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compute_instability(explanations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = all_data.drop(\"type_data\", axis=1)\n",
    "y = all_data[\"type_data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0)\n",
    "clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "exstream-venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}