m365_ioc_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "669832c64f4a4eb189948992d705588b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Dropdown(description='Top N stats', index=4, options=('10', '20', '30', '40', '50'), style=DescriptionStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b4cb37f22cb4e54b14afd3b59305904",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ToggleButtons(button_style='info', description='Projection type:', layout=Layout(width='520px'), options=('Dow…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48077091493f4139b3b27fc85736b9ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntRangeSlider(value=(0, 168), description='Time range in hours (max 1 week from the first event)', layout=Lay…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3520a942a9d84b2bb7953f768e4144cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(button_style='success', description='Apply changes', style=ButtonStyle(), tooltip='Apply changes')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcf3b2ae4e564a85ab84c019c7181ca4",
       "version_major": 2,
       "version_minor": 1
      },
      "text/plain": [
       "JupyterChart(spec={'config': {'view': {'continuousWidth': 300, 'continuousHeight': 300}}, 'data': {'name': 'da…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "import panel as pn\n",
    "import ipywidgets as widgets\n",
    "import pandas as pd\n",
    "import altair as alt\n",
    "import pathlib\n",
    "import datetime\n",
    "from vega_datasets import data\n",
    "import os\n",
    "import pycountry\n",
    "\n",
    "TIMESTAMP_START = \"First Downloaded At\"\n",
    "TIMESTAMP_END = \"Last Downloaded At\"\n",
    "COUNT = \"COUNT\"\n",
    "IP_ADDR = \"Client IP\"\n",
    "DATA_AMOUNT = \"File Size (MB)\"\n",
    "FILE_NAME = \"Filename\"\n",
    "URL = \"URL\"\n",
    "DOWNLOAD_COUNT = \"Download Count\"\n",
    "APP = \"Application\"\n",
    "FILE_EXT = \"File Extension\"\n",
    "\n",
    "\n",
    "#User Options\n",
    "DATA_AMOUNT_DSC = \"Downloaded data amount\"\n",
    "TIME_PLOT_DSC = \"Time plot\"\n",
    "\n",
    "\n",
    "prefix_file_path = os.path.join(\"Task 2\", \"536535_data\", \"536535_data\")\n",
    "ioc_file_path = os.path.join(prefix_file_path, \"m365_ioc_search\", \"ioc_events.csv\")\n",
    "dir_path = pathlib.Path().resolve()\n",
    "\n",
    "ioc_data: pd.DataFrame = pd.DataFrame(pd.read_csv(dir_path / ioc_file_path))\n",
    "\n",
    "pn.extension('vega')\n",
    "\n",
    "style = {'description_width': 'initial'}\n",
    "\n",
    "top_n_widget = widgets.Dropdown(\n",
    "    options=['10', '20', '30', '40', '50'],\n",
    "    value='10',\n",
    "    description='Top N stats',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    ")\n",
    "\n",
    "data_type_widget = widgets.ToggleButtons(\n",
    "    options=[DATA_AMOUNT_DSC, TIME_PLOT_DSC],\n",
    "    description='Projection type:',\n",
    "    disabled=False,\n",
    "    button_style='info', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style = {'button_width':'200px'}, layout={'width': '520px'},\n",
    ")\n",
    "\n",
    "apply_changes_widget = widgets.Button(\n",
    "    description='Apply changes',\n",
    "    tooltip='Apply changes',\n",
    "    disabled=False,\n",
    "    button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style = style,\n",
    ")\n",
    "period_widget = widgets.IntRangeSlider(\n",
    "    min=0,\n",
    "    max=168,\n",
    "    value=[0, 168],\n",
    "    step=1,\n",
    "    description=\"Time range in hours (max 1 week from the first event)\",\n",
    "    disabled=False,\n",
    "    style = style,\n",
    "    layout={'width': '520px'}\n",
    ")\n",
    "\n",
    "ui_widgets = [top_n_widget, data_type_widget, \n",
    "              period_widget, apply_changes_widget,\n",
    "              ]\n",
    "\n",
    "def display_widgets():\n",
    "    for ui_widget in ui_widgets:\n",
    "        display(ui_widget)\n",
    "\n",
    "def get_first_time_flow_string():\n",
    "    return download_data[TIMESTAMP_START].min()\n",
    "\n",
    "def get_last_time_flow_string():\n",
    "    return download_data[TIMESTAMP_END].max()\n",
    "\n",
    "def get_first_time_flow():\n",
    "    return datetime.datetime.fromisoformat(get_first_time_flow_string())\n",
    "\n",
    "def get_last_time_flow():\n",
    "    return datetime.datetime.fromisoformat(get_last_time_flow_string())\n",
    "\n",
    "def to_datetime(x):\n",
    "    return datetime.datetime.fromisoformat(x)\n",
    "\n",
    "def process_by_top_n(current_data, top_n):\n",
    "    if current_data.empty:\n",
    "        return current_data\n",
    "    return current_data.nlargest(top_n, columns=DATA_AMOUNT)\n",
    "\n",
    "def process_by_time_range(current_data, time_period):\n",
    "    if current_data.empty:\n",
    "        return current_data\n",
    "    \n",
    "    start_offset, end_offset = time_period\n",
    "\n",
    "\n",
    "    start = get_first_time_flow() + datetime.timedelta(hours=start_offset)\n",
    "    end = get_first_time_flow() + datetime.timedelta(hours=end_offset)\n",
    "\n",
    "    return current_data[(current_data[TIMESTAMP_START].apply(to_datetime) >= start) &\n",
    "                      ((current_data[TIMESTAMP_END].apply(to_datetime)) <= end)]\n",
    "\n",
    "#downloaded data amount is not multiplied by Download count\n",
    "def process_data_download(current_data: pd.DataFrame, top_n, time_period):\n",
    "    processed_data = process_by_time_range(current_data, time_period)\n",
    "    processed_data = processed_data.groupby(FILE_NAME, as_index=False).aggregate({DATA_AMOUNT: 'sum', DOWNLOAD_COUNT: 'sum', \n",
    "                                                                                  FILE_EXT: 'first', APP: 'first'})\n",
    "    return process_by_top_n(processed_data, top_n)\n",
    "\n",
    "def make_download_plot(current_data):\n",
    "    plot = alt.Chart(current_data).mark_bar().encode(\n",
    "        x=alt.X(DATA_AMOUNT),\n",
    "        y=alt.Y(FILE_NAME, sort='-x'),\n",
    "        color=alt.Color(DATA_AMOUNT,\n",
    "                   scale=alt.Scale(range=['lightgreen', 'green'])),\n",
    "         tooltip=[\n",
    "                 alt.Tooltip(f\"{FILE_NAME}\", title=\"File name\"),\n",
    "                 alt.Tooltip(f\"{FILE_EXT}\", title=\"File extension\"),\n",
    "                 alt.Tooltip(f\"{APP}\", title=\"Download via\"),\n",
    "                 alt.Tooltip(f\"{DATA_AMOUNT}:Q\", title=\"Downloaded data amout\"),\n",
    "                 alt.Tooltip(f\"{DOWNLOAD_COUNT}:Q\", title=\"Download count\"),\n",
    "             ],\n",
    "    ).properties(\n",
    "        title=f\"File size download\",    \n",
    "        width=800, \n",
    "        height=500,\n",
    "    )\n",
    "    return alt.JupyterChart(plot)\n",
    "    \n",
    "    \n",
    "def on_change(v):\n",
    "    clear_output(wait=True)\n",
    "    top_n = int(top_n_widget.value)\n",
    "    display_widgets()\n",
    "\n",
    "    if data_type_widget.value == DATA_AMOUNT_DSC:\n",
    "        processed_data = process_data_download(download_data, top_n, period_widget.value)\n",
    "    \n",
    "    if processed_data.empty:\n",
    "        print(\"Log: The data set is empty\")\n",
    "        return\n",
    "    \n",
    "    if (data_type_widget.value == DATA_AMOUNT_DSC):\n",
    "        plot: alt.JupyterChart = make_download_plot(processed_data)\n",
    "\n",
    "    \n",
    "    display(plot)\n",
    "\n",
    "\n",
    "display_widgets()\n",
    "apply_changes_widget.on_click(on_change)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}