TOP-N.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script type=\"esms-options\">{\"shimMode\": true}</script><style>*[data-root-id],\n",
       "*[data-root-id] > * {\n",
       "  box-sizing: border-box;\n",
       "  font-family: var(--jp-ui-font-family);\n",
       "  font-size: var(--jp-ui-font-size1);\n",
       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
       "}\n",
       "\n",
       "/* Override VSCode background color */\n",
       ".cell-output-ipywidget-background:has(\n",
       "    > .cell-output-ipywidget-background > .lm-Widget > *[data-root-id]\n",
       "  ),\n",
       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
       "  background-color: transparent !important;\n",
       "}\n",
       "</style>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  const force = true;\n  const py_version = '3.6.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  const reloading = false;\n  const Bokeh = root.Bokeh;\n\n  // Set a timeout for this load but only if we are not already initializing\n  if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      // Don't load bokeh if it is still initializing\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      // There is nothing to load\n      run_callbacks();\n      return null;\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error(e) {\n      const src_el = e.srcElement\n      console.error(\"failed to load \" + (src_el.href || src_el.src));\n    }\n\n    const skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'vega-embed': 'https://cdn.jsdelivr.net/npm/vega-embed@6/build/vega-embed.min', 'vega-lite': 'https://cdn.jsdelivr.net/npm/vega-lite@5/build/vega-lite.min', 'vega': 'https://cdn.jsdelivr.net/npm/vega@5/build/vega.min'}, 'shim': {}});\n      require([\"vega-embed\"], function(vegaEmbed) {\n        window.vegaEmbed = vegaEmbed\n        on_load()\n      })\n      require([\"vega-lite\"], function(vl) {\n        window.vl = vl\n        on_load()\n      })\n      require([\"vega\"], function(vega) {\n        window.vega = vega\n        on_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 3;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    const existing_stylesheets = []\n    const links = document.getElementsByTagName('link')\n    for (let i = 0; i < links.length; i++) {\n      const link = links[i]\n      if (link.href != null) {\n        existing_stylesheets.push(link.href)\n      }\n    }\n    for (let i = 0; i < css_urls.length; i++) {\n      const url = css_urls[i];\n      const escaped = encodeURI(url)\n      if (existing_stylesheets.indexOf(escaped) !== -1) {\n        on_load()\n        continue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window.vega !== undefined) && (!(window.vega instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega@5'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(encodeURI(urls[i]))\n      }\n    }    if (((window.vegaLite !== undefined) && (!(window.vegaLite instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega-lite@5'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(encodeURI(urls[i]))\n      }\n    }    if (((window.vegaEmbed !== undefined) && (!(window.vegaEmbed instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega-embed@6'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(encodeURI(urls[i]))\n      }\n    }    var existing_scripts = []\n    const scripts = document.getElementsByTagName('script')\n    for (let i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n        existing_scripts.push(script.src)\n      }\n    }\n    for (let i = 0; i < js_urls.length; i++) {\n      const url = js_urls[i];\n      const escaped = encodeURI(url)\n      if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n        if (!window.requirejs) {\n          on_load();\n        }\n        continue;\n      }\n      const element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (let i = 0; i < js_modules.length; i++) {\n      const url = js_modules[i];\n      const escaped = encodeURI(url)\n      if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n        if (!window.requirejs) {\n          on_load();\n        }\n        continue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      const url = js_exports[name];\n      const escaped = encodeURI(url)\n      if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n        if (!window.requirejs) {\n          on_load();\n        }\n        continue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  const js_urls = [\"https://cdn.holoviz.org/panel/1.5.5/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega@5\", \"https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega-lite@5\", \"https://cdn.holoviz.org/panel/1.5.5/dist/bundled/vegaplot/vega-embed@6\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.6.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.6.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.6.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.6.2.min.js\", \"https://cdn.holoviz.org/panel/1.5.5/dist/panel.min.js\"];\n  const js_modules = [];\n  const js_exports = {};\n  const css_urls = [];\n  const inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (let i = 0; i < inline_js.length; i++) {\n        try {\n          inline_js[i].call(root, root.Bokeh);\n        } catch(e) {\n          if (!reloading) {\n            throw e;\n          }\n        }\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n        var NewBokeh = root.Bokeh;\n        if (Bokeh.versions === undefined) {\n          Bokeh.versions = new Map();\n        }\n        if (NewBokeh.version !== Bokeh.version) {\n          Bokeh.versions.set(NewBokeh.version, NewBokeh)\n        }\n        root.Bokeh = Bokeh;\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      // If the timeout and bokeh was not successfully loaded we reset\n      // everything and try loading again\n      root._bokeh_timeout = Date.now() + 5000;\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      root._bokeh_is_loading = 0\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      const bokeh_loaded = root.Bokeh != null && (root.Bokeh.version === py_version || (root.Bokeh.versions !== undefined && root.Bokeh.versions.has(py_version)));\n      if (!reloading && !bokeh_loaded) {\n        if (root.Bokeh) {\n          root.Bokeh = undefined;\n        }\n        console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n        console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n        run_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
      "application/vnd.holoviews_load.v0+json": ""
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        })\n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
      "application/vnd.holoviews_load.v0+json": ""
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.holoviews_exec.v0+json": "",
      "text/html": [
       "<div id='96824e88-094b-490d-a65e-852371029126'>\n",
       "  <div id=\"ce54a8c0-227c-46cd-b9d6-1c6c5e4ee727\" data-root-id=\"96824e88-094b-490d-a65e-852371029126\" style=\"display: contents;\"></div>\n",
       "</div>\n",
       "<script type=\"application/javascript\">(function(root) {\n",
       "  var docs_json = {\"cf4d84e0-e8bb-4169-9f74-cbdc8d695aa0\":{\"version\":\"3.6.2\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"panel.models.browser.BrowserInfo\",\"id\":\"96824e88-094b-490d-a65e-852371029126\"},{\"type\":\"object\",\"name\":\"panel.models.comm_manager.CommManager\",\"id\":\"03efce25-0356-4dfb-bc43-a4d321096fe6\",\"attributes\":{\"plot_id\":\"96824e88-094b-490d-a65e-852371029126\",\"comm_id\":\"e2d30df1ca3847129ded9995cbd66bdc\",\"client_comm_id\":\"4eedb5f94ff14c6fb6c9e4557459d066\"}}],\"defs\":[{\"type\":\"model\",\"name\":\"ReactiveHTML1\"},{\"type\":\"model\",\"name\":\"FlexBox1\",\"properties\":[{\"name\":\"align_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"align_items\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"flex_direction\",\"kind\":\"Any\",\"default\":\"row\"},{\"name\":\"flex_wrap\",\"kind\":\"Any\",\"default\":\"wrap\"},{\"name\":\"gap\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"justify_content\",\"kind\":\"Any\",\"default\":\"flex-start\"}]},{\"type\":\"model\",\"name\":\"FloatPanel1\",\"properties\":[{\"name\":\"config\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"contained\",\"kind\":\"Any\",\"default\":true},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"right-top\"},{\"name\":\"offsetx\",\"kind\":\"Any\",\"default\":null},{\"name\":\"offsety\",\"kind\":\"Any\",\"default\":null},{\"name\":\"theme\",\"kind\":\"Any\",\"default\":\"primary\"},{\"name\":\"status\",\"kind\":\"Any\",\"default\":\"normalized\"}]},{\"type\":\"model\",\"name\":\"GridStack1\",\"properties\":[{\"name\":\"mode\",\"kind\":\"Any\",\"default\":\"warn\"},{\"name\":\"ncols\",\"kind\":\"Any\",\"default\":null},{\"name\":\"nrows\",\"kind\":\"Any\",\"default\":null},{\"name\":\"allow_resize\",\"kind\":\"Any\",\"default\":true},{\"name\":\"allow_drag\",\"kind\":\"Any\",\"default\":true},{\"name\":\"state\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"drag1\",\"properties\":[{\"name\":\"slider_width\",\"kind\":\"Any\",\"default\":5},{\"name\":\"slider_color\",\"kind\":\"Any\",\"default\":\"black\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":50}]},{\"type\":\"model\",\"name\":\"click1\",\"properties\":[{\"name\":\"terminal_output\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"debug_name\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"clears\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"FastWrapper1\",\"properties\":[{\"name\":\"object\",\"kind\":\"Any\",\"default\":null},{\"name\":\"style\",\"kind\":\"Any\",\"default\":null}]},{\"type\":\"model\",\"name\":\"NotificationAreaBase1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"NotificationArea1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"notifications\",\"kind\":\"Any\",\"default\":[]},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0},{\"name\":\"types\",\"kind\":\"Any\",\"default\":[{\"type\":\"map\",\"entries\":[[\"type\",\"warning\"],[\"background\",\"#ffc107\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-exclamation-triangle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]},{\"type\":\"map\",\"entries\":[[\"type\",\"info\"],[\"background\",\"#007bff\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-info-circle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]}]}]},{\"type\":\"model\",\"name\":\"Notification\",\"properties\":[{\"name\":\"background\",\"kind\":\"Any\",\"default\":null},{\"name\":\"duration\",\"kind\":\"Any\",\"default\":3000},{\"name\":\"icon\",\"kind\":\"Any\",\"default\":null},{\"name\":\"message\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"notification_type\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_rendered\",\"kind\":\"Any\",\"default\":false},{\"name\":\"_destroyed\",\"kind\":\"Any\",\"default\":false}]},{\"type\":\"model\",\"name\":\"TemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"BootstrapTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"TemplateEditor1\",\"properties\":[{\"name\":\"layout\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"MaterialTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"ReactiveESM1\",\"properties\":[{\"name\":\"esm_constants\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}}]},{\"type\":\"model\",\"name\":\"JSComponent1\",\"properties\":[{\"name\":\"esm_constants\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}}]},{\"type\":\"model\",\"name\":\"ReactComponent1\",\"properties\":[{\"name\":\"esm_constants\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}}]},{\"type\":\"model\",\"name\":\"AnyWidgetComponent1\",\"properties\":[{\"name\":\"esm_constants\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}}]},{\"type\":\"model\",\"name\":\"request_value1\",\"properties\":[{\"name\":\"fill\",\"kind\":\"Any\",\"default\":\"none\"},{\"name\":\"_synced\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_request_sync\",\"kind\":\"Any\",\"default\":0}]}]}};\n",
       "  var render_items = [{\"docid\":\"cf4d84e0-e8bb-4169-9f74-cbdc8d695aa0\",\"roots\":{\"96824e88-094b-490d-a65e-852371029126\":\"ce54a8c0-227c-46cd-b9d6-1c6c5e4ee727\"},\"root_ids\":[\"96824e88-094b-490d-a65e-852371029126\"]}];\n",
       "  var docs = Object.values(docs_json)\n",
       "  if (!docs) {\n",
       "    return\n",
       "  }\n",
       "  const py_version = docs[0].version.replace('rc', '-rc.').replace('.dev', '-dev.')\n",
       "  async function embed_document(root) {\n",
       "    var Bokeh = get_bokeh(root)\n",
       "    await Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "    for (const render_item of render_items) {\n",
       "      for (const root_id of render_item.root_ids) {\n",
       "\tconst id_el = document.getElementById(root_id)\n",
       "\tif (id_el.children.length && id_el.children[0].hasAttribute('data-root-id')) {\n",
       "\t  const root_el = id_el.children[0]\n",
       "\t  root_el.id = root_el.id + '-rendered'\n",
       "\t  for (const child of root_el.children) {\n",
       "            // Ensure JupyterLab does not capture keyboard shortcuts\n",
       "            // see: https://jupyterlab.readthedocs.io/en/4.1.x/extension/notebook.html#keyboard-interaction-model\n",
       "\t    child.setAttribute('data-lm-suppress-shortcuts', 'true')\n",
       "\t  }\n",
       "\t}\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  function get_bokeh(root) {\n",
       "    if (root.Bokeh === undefined) {\n",
       "      return null\n",
       "    } else if (root.Bokeh.version !== py_version) {\n",
       "      if (root.Bokeh.versions === undefined || !root.Bokeh.versions.has(py_version)) {\n",
       "\treturn null\n",
       "      }\n",
       "      return root.Bokeh.versions.get(py_version);\n",
       "    } else if (root.Bokeh.version === py_version) {\n",
       "      return root.Bokeh\n",
       "    }\n",
       "    return null\n",
       "  }\n",
       "  function is_loaded(root) {\n",
       "    var Bokeh = get_bokeh(root)\n",
       "    return (Bokeh != null && Bokeh.Panel !== undefined && ( root.vega !== undefined) && ( root.vega !== undefined))\n",
       "  }\n",
       "  if (is_loaded(root)) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (is_loaded(root)) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else if (document.readyState == \"complete\") {\n",
       "        attempts++;\n",
       "        if (attempts > 200) {\n",
       "          clearInterval(timer);\n",
       "\t  var Bokeh = get_bokeh(root)\n",
       "\t  if (Bokeh == null || Bokeh.Panel == null) {\n",
       "            console.warn(\"Panel: ERROR: Unable to run Panel code because Bokeh or Panel library is missing\");\n",
       "\t  } else {\n",
       "\t    console.warn(\"Panel: WARNING: Attempting to render but not all required libraries could be resolved.\")\n",
       "\t    embed_document(root)\n",
       "\t  }\n",
       "        }\n",
       "      }\n",
       "    }, 25, root)\n",
       "  }\n",
       "})(window);</script>"
      ]
     },
     "metadata": {
      "application/vnd.holoviews_exec.v0+json": {
       "id": "96824e88-094b-490d-a65e-852371029126"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34ad2791b2ae4ac18ac52e82ff4597cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Dropdown(description='Top N stats', options=('10', '20', '30', '40'), style=DescriptionStyle(description_width…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e893053bb19a44e49bd182a789c61c76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ToggleButtons(button_style='info', description='Info about single IP addresss or pair:', options=('Single', 'P…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f98b161214548be9170625ff2ea9323",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ToggleButtons(button_style='info', description='Direction:', options=('Received', 'Sent'), style=ToggleButtons…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22342aa53325405b9afd81518a503f4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ToggleButtons(button_style='info', description='flow_data type:', layout=Layout(width='520px'), options=('Byte…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0e48d7c3d44452f91d1bf2c69da2622",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Text(value='disabled', description='Minimal duration of a flow in seconds', disabled=True, layout=Layout(width…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d048a5447f94bc4b8d0d48565a4ad35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntRangeSlider(value=(0, 36000), description='Time range:', max=36000, step=5, style=SliderStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57a0bbe6f3e94be38299263390eb8744",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Textarea(value='0.0.0.0/0,', description='List of IPv4 masks, separeted by \",\"', layout=Layout(width='50%'), s…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc2ea2e940354a098bdd415317f11407",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Textarea(value='0:0:0:0:0:0:0:0/0,', description='List of IPv6 masks, separeted by \",\"', layout=Layout(width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e727c3222084b7d8c10156e6d2836d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "SelectMultiple(description='Protocol', index=(0,), options=('ALL', 'TCP', 'UDP', 'SMTP', 'DNS', 'HTTP', 'HTTPS…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf92e5565d1b453b96b17e131d7780d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Dropdown(description='Ports', options=('ALL', '8080, 8888, 591, 82', '8443, 9443, 4443', '5353, 5355', '2121, …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f63f7499fb874c05a561894807313aea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(button_style='success', description='Apply changes', style=ButtonStyle(), tooltip='Apply changes')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "from IPython.display import clear_output\n",
    "from ipaddress import ip_address, ip_network, IPv4Address, IPv6Address\n",
    "\n",
    "import panel as pn\n",
    "import ipywidgets as widgets\n",
    "import pandas as pd\n",
    "import altair as alt\n",
    "import pathlib\n",
    "import datetime\n",
    "import os\n",
    "\n",
    "pn.extension('vega')\n",
    "\n",
    "SRC_AND_DST_IP = [\"SOURCE IP ADDRESS\", \"DESTINATION IP ADDRESS\"]\n",
    "\n",
    "SRC_IP = SRC_AND_DST_IP[0]\n",
    "DST_IP = SRC_AND_DST_IP[1]\n",
    "\n",
    "DATA_TYPE_DSC = {\"Bytes\": \"BYTES\", \"Packets\": \"PACKETS\",\n",
    "                 \"Flows\": \"FLOWS\", \"Subdomains of muni.cz\": \"DNS QNAME\",\n",
    "                 \"Unusual TLDs\": \"UNUSUAL TLD\", \n",
    "                 \"IP requesting unusual TLD\": \"IP REQUESTING UNUSUAL TLD\",\n",
    "                 \"Most DNS requests\": \"MOST DNS REQUESTS\", \n",
    "                 'Duration with dst IP': \"DURATION\", \n",
    "                 \"DNS communication\": \"DNS_COMM\", \"SMTP communication\": \"SMTP_COMM\"}\n",
    "\n",
    "SINGLE_PAIR_DSC = {\"Pair\" : \"PAIR\",\n",
    "                   \"Single\": \"SINGLE\"}\n",
    "\n",
    "COUNT = \"COUNT\"\n",
    "\n",
    "file_path = os.path.join(\"Task 1\", \"20240701_16-17_100l.csv\")\n",
    "dir_path = pathlib.Path().resolve()\n",
    "\n",
    "csv_file = pd.read_csv(dir_path / file_path)\n",
    "flow_data: pd.DataFrame = pd.DataFrame(csv_file)\n",
    "\n",
    "def get_first_time_flow_string():\n",
    "    return flow_data[\"START TIME - FIRST SEEN\"].min()\n",
    "\n",
    "def get_last_time_flow_string():\n",
    "    return flow_data[\"START TIME - FIRST SEEN\"].max()\n",
    "\n",
    "def get_first_time_flow():\n",
    "    return datetime.datetime.fromisoformat(get_first_time_flow_string())\n",
    "\n",
    "def get_last_time_flow():\n",
    "    return datetime.datetime.fromisoformat(get_last_time_flow_string())\n",
    "\n",
    "\n",
    "def ipv4_in_range_cmp(ip, min, max):\n",
    "    if is_ipv4(ip):\n",
    "        return min <= try_convert_ip_to_int(ip) <= max\n",
    "    return True\n",
    "\n",
    "def ipv6_in_range_cmp(ip, min, max):\n",
    "    if is_ipv6(ip):\n",
    "        return min <= try_convert_ip_to_int(ip) <= max\n",
    "    return True    \n",
    "\n",
    "def try_convert_ip_to_int(ip):\n",
    "    try:\n",
    "        return int(ip_address(ip))\n",
    "    except Exception:\n",
    "        return 0\n",
    "\n",
    "def is_ipv6(addr):\n",
    "    try:\n",
    "        return type(ip_address(addr)) is IPv6Address\n",
    "    except Exception:\n",
    "        return False\n",
    "\n",
    "def is_ipv4(addr):\n",
    "    try:\n",
    "        return type(ip_address(addr)) is IPv4Address\n",
    "    except Exception:\n",
    "        return False\n",
    "\n",
    "def to_datetime(x):\n",
    "    return datetime.datetime.fromisoformat(x)\n",
    "\n",
    "def time_to_seconds(time_str):\n",
    "    time_parts = time_str.split()\n",
    "    minutes = int(time_parts[0].strip('m'))\n",
    "    seconds = int(time_parts[1].strip('s'))\n",
    "\n",
    "    total_minutes = minutes * 60 + seconds\n",
    "    return total_minutes\n",
    "\n",
    "def is_in_muni_subdomain(input_str):\n",
    "    return \"muni.cz\" in input_str\n",
    "\n",
    "style = {'description_width': 'initial'}\n",
    "\n",
    "apply_changes_widget = widgets.Button(\n",
    "    description='Apply changes',\n",
    "    tooltip='Apply changes',\n",
    "    disabled=False,\n",
    "    button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style = style,\n",
    ")\n",
    "\n",
    "top_n_widget = widgets.Dropdown(\n",
    "    options=['10', '20', '30', '40'],\n",
    "    value='10',\n",
    "    description='Top N stats',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    ")\n",
    "\n",
    "single_or_pair_widget = widgets.ToggleButtons(\n",
    "    options=['Single', 'Pair'],\n",
    "    description='Info about single IP addresss or pair:',\n",
    "    disabled=False,\n",
    "    button_style='info', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style = style,\n",
    ")\n",
    "\n",
    "src_dst_widget = widgets.ToggleButtons(\n",
    "    options=['Received', 'Sent'],\n",
    "    description='Direction:',\n",
    "    disabled=False,\n",
    "    button_style='info', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style=style,\n",
    ")\n",
    "\n",
    "data_type_widget = widgets.ToggleButtons(\n",
    "    options=list(DATA_TYPE_DSC.keys()),\n",
    "    description='flow_data type:',\n",
    "    disabled=False,\n",
    "    button_style='info', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    style = {'button_width':'200px'}, \n",
    "    layout={'width': '520px'},\n",
    ")\n",
    "\n",
    "period_widget = widgets.IntRangeSlider(\n",
    "    min=0,\n",
    "    max=36000,\n",
    "    value=[0, 36000],\n",
    "    step=5,\n",
    "    description=\"Time range:\",\n",
    "    disabled=False,\n",
    "    style = style,\n",
    ")\n",
    "\n",
    "ipv4_range_widget = widgets.Textarea(\n",
    "    value='0.0.0.0/0,',\n",
    "    description='List of IPv4 masks, separeted by \\\",\\\"',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    "    layout = widgets.Layout(width='50%')\n",
    ")\n",
    "\n",
    "ipv6_range_widget = widgets.Textarea(\n",
    "    value='0:0:0:0:0:0:0:0/0,',\n",
    "    description='List of IPv6 masks, separeted by \\\",\\\"',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    "    layout = widgets.Layout(width='80%')\n",
    ")\n",
    "\n",
    "protocol_widget = widgets.SelectMultiple(\n",
    "    options=[\"ALL\", \"TCP\", \"UDP\", \"SMTP\",\n",
    "             \"DNS\", \"HTTP\", \"HTTPS\",\n",
    "             \"SMB\", \"TELNET\",\n",
    "             \"RDP\", \"FTP\", \"TFTFP\"\n",
    "            ],\n",
    "    value=(\"ALL\",),\n",
    "    description='Protocol',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    "    rows=12,\n",
    ")\n",
    "\n",
    "port_widget = widgets.Dropdown(\n",
    "    options=[\"ALL\",\n",
    "             \"8080, 8888, 591, 82\",\n",
    "             \"8443, 9443, 4443\",\n",
    "             \"5353, 5355\",\n",
    "             \"2121, 8021\",\n",
    "             \"2525, 26\",\n",
    "             \"3390, 3391, 4000\",\n",
    "            ],\n",
    "    description='Ports',\n",
    "    disabled=False,\n",
    "    style = style,\n",
    "    layout={'Height': '520px'}\n",
    ")\n",
    "\n",
    "minimal_duration_widget = widgets.Text(\n",
    "    value='disabled',\n",
    "    description='Minimal duration of a flow in seconds',\n",
    "    disabled=True,\n",
    "    style = style,\n",
    "    layout = widgets.Layout(width='50%')\n",
    ")\n",
    "\n",
    "table_widget = widgets.HTML(value=flow_data.iloc[:0].to_html())\n",
    "\n",
    "ui_widgets = [\n",
    "              top_n_widget, single_or_pair_widget, src_dst_widget, data_type_widget,\n",
    "              minimal_duration_widget, period_widget, ipv4_range_widget, ipv6_range_widget,\n",
    "              protocol_widget, port_widget, apply_changes_widget,\n",
    "              ]\n",
    "\n",
    "def display_widgets():\n",
    "    for ui_widget in ui_widgets:\n",
    "        display(ui_widget)\n",
    "\n",
    "def disable_button(button):\n",
    "    button.button_style = ''\n",
    "    button.disabled = True\n",
    "\n",
    "def enable_button(button):\n",
    "    button.button_style = 'info'\n",
    "    button.disabled = False\n",
    "\n",
    "def process_flow_pair(curr_data, top_n_n):\n",
    "    tmp_data = curr_data[[SRC_IP, DST_IP]].apply(lambda x: tuple(sorted(x)), axis=1).value_counts().reset_index()\n",
    "    tmp_data.columns = ['pair', DATA_TYPE_DSC[\"Flows\"]]\n",
    "    tmp_data[[SRC_IP, DST_IP]] = pd.DataFrame(tmp_data['pair'].tolist(), index=tmp_data.index)\n",
    "    processed_data = tmp_data.drop(columns=['pair']).nlargest(top_n_n, columns=DATA_TYPE_DSC[\"Flows\"])\n",
    "    processed_data['src-dst'] = processed_data[SRC_IP] + ' - ' + processed_data[DST_IP]\n",
    "    return processed_data[['src-dst', DATA_TYPE_DSC[\"Flows\"]]]\n",
    "\n",
    "def process_duration_single(curr_data, g_data_type, minimal_duration, top_n_n):\n",
    "    processed_data = pd.DataFrame(curr_data)\n",
    "    processed_data[g_data_type] = processed_data[g_data_type].apply(time_to_seconds)\n",
    "    return processed_data[(processed_data[g_data_type] >= int(minimal_duration))].groupby(DST_IP).agg(\n",
    "                    DURATION=(g_data_type, 'sum'),\n",
    "                    COUNT=(DST_IP, 'count'),\n",
    "                    ).reset_index().nlargest(top_n_n, columns=g_data_type)\n",
    "\n",
    "def process_ip_requesting_unusual_tld(curr_data, g_data_type, top_n_n, unusual_tlds_regex):\n",
    "    processed_data = curr_data[curr_data[g_data_type].str.contains(unusual_tlds_regex, regex=True, na=False)]\n",
    "    return processed_data.groupby(SRC_IP)[SRC_IP].count().reset_index(name=\"IP REQUESTING UNUSUAL TLD\").nlargest(top_n_n, columns=\"IP REQUESTING UNUSUAL TLD\")\n",
    "      \n",
    "def process_unusual_tld(curr_data, g_data_type, top_n_n, unusual_tlds_regex):\n",
    "    processed_data = curr_data[curr_data[g_data_type].str.contains(unusual_tlds_regex, regex=True, na=False)]\n",
    "    processed_data = processed_data.groupby(g_data_type)[g_data_type].count().reset_index(name=COUNT).nlargest(top_n_n, columns=COUNT)\n",
    "    return processed_data.rename(columns={g_data_type: \"UNUSUAL TLD\"})\n",
    "\n",
    "def process_by_type(curr_data, g_src_dst, g_data_type, top_n_n, single_or_pair, minimal_duration):\n",
    "    if curr_data.empty:\n",
    "        return curr_data\n",
    "\n",
    "    unusual_tlds_regex = \"\\.xyz$|\\.top$|\\.club$|\\.site$|\\.online$|\\.loan$|\\.trade$|\\.accountants$|\\.space$\"\n",
    "    if single_or_pair == \"Pair\":\n",
    "        tmp_data = pd.DataFrame(curr_data)\n",
    "        if g_data_type == \"FLOWS\":\n",
    "            processed_data = process_flow_pair(curr_data, top_n_n)\n",
    "        else:\n",
    "            tmp_data['pair'] = curr_data[[SRC_IP, DST_IP]].apply(lambda x: tuple(sorted(x)), axis=1)\n",
    "            processed_data = tmp_data.groupby('pair', as_index=False)[g_data_type].sum().nlargest(top_n_n, columns=g_data_type)\n",
    "\n",
    "    else:\n",
    "        if g_data_type == \"FLOWS\":\n",
    "            processed_data = curr_data.groupby(g_src_dst)[g_src_dst].count().reset_index(name=g_data_type).nlargest(top_n_n, columns=g_data_type)\n",
    "        elif g_data_type == \"DURATION\":\n",
    "            processed_data = process_duration_single(curr_data, g_data_type, minimal_duration, top_n_n)\n",
    "        elif g_data_type == \"DNS QNAME\":\n",
    "            processed_data = curr_data[curr_data[g_data_type].str.contains('muni.cz', na=False)]\n",
    "            processed_data = processed_data.groupby(g_data_type)[g_data_type].count().reset_index(name=COUNT).nlargest(top_n_n, columns=COUNT)\n",
    "        elif g_data_type == \"UNUSUAL TLD\":\n",
    "            processed_data = process_unusual_tld(curr_data, \"DNS QNAME\", top_n_n, unusual_tlds_regex)\n",
    "        elif g_data_type == \"IP REQUESTING UNUSUAL TLD\":\n",
    "            processed_data = process_ip_requesting_unusual_tld(curr_data, \"DNS QNAME\", top_n_n, unusual_tlds_regex)\n",
    "        elif g_data_type == \"MOST DNS REQUESTS\":\n",
    "            processed_data = curr_data[curr_data['DNS QNAME'].notna()].groupby(SRC_IP)[SRC_IP].count().reset_index(name=g_data_type).nlargest(top_n_n, columns=g_data_type)\n",
    "        elif g_data_type in [\"SMTP_COMM\", \"DNS_COMM\"]:\n",
    "            processed_data = process_by_dns_smtp(curr_data, g_data_type, top_n_n)\n",
    "\n",
    "        else:\n",
    "            processed_data = curr_data[[g_src_dst, g_data_type]].groupby(g_src_dst).sum().reset_index().nlargest(top_n_n, columns=g_data_type)\n",
    "            processed_data = processed_data.merge(flow_data.groupby(g_src_dst)[g_src_dst].count().reset_index(name=COUNT), how='left', on=g_src_dst)\n",
    "\n",
    "    return processed_data\n",
    "\n",
    "def process_by_ipvx_range_pair(curr_data, ip_ranges, cmp_func):\n",
    "    tmp_data = pd.DataFrame()\n",
    "    for ip_subnet in ip_ranges:\n",
    "        if len(ip_subnet) <= 1:\n",
    "            continue\n",
    "        ip_range_int = ip_network(ip_subnet.strip())\n",
    "        ip_start, ip_end = int(ip_range_int[0]), int(ip_range_int[-1])\n",
    "    \n",
    "        tmp_data = pd.concat([tmp_data, curr_data[(curr_data[SRC_IP].apply(cmp_func, args=(ip_start, ip_end))) &\n",
    "                                                    (curr_data[DST_IP].apply(cmp_func, args=(ip_start, ip_end)))]])\n",
    "    \n",
    "    return tmp_data\n",
    "\n",
    "#need to fix filtering ipv4 and ipv6 addresses separately\n",
    "def process_by_ipvx_range_single(curr_data, g_src_dst, ip_ranges):\n",
    "    tmp_data = pd.DataFrame()\n",
    "    for ip_subnet in ip_ranges:\n",
    "        if len(ip_subnet) <= 1:\n",
    "            continue\n",
    "        ip_range_int = ip_network(ip_subnet.strip())\n",
    "        ip_start, ip_end = int(ip_range_int[0]), int(ip_range_int[-1])\n",
    "\n",
    "        tmp_data = pd.concat([tmp_data, curr_data[(curr_data[g_src_dst].apply(try_convert_ip_to_int).between(ip_start, ip_end))]])\n",
    "\n",
    "    return pd.DataFrame(tmp_data)\n",
    "\n",
    "def process_by_ip_range(curr_data, g_src_dst, ipv4_ranges_str, ipv6_ranges_str, single_or_pair):\n",
    "    if curr_data.empty:\n",
    "        return curr_data\n",
    "\n",
    "    if single_or_pair == 'Single':\n",
    "        ipv4_addrs = curr_data[(curr_data[g_src_dst].apply(is_ipv4))]\n",
    "        ipv6_addrs = curr_data[(curr_data[g_src_dst].apply(is_ipv6))]\n",
    "        return pd.concat([process_by_ipvx_range_single(ipv4_addrs, g_src_dst, ipv4_ranges_str), \n",
    "                        process_by_ipvx_range_single(ipv6_addrs, g_src_dst, ipv6_ranges_str),\n",
    "                        ])\n",
    "\n",
    "    ipv4_addrs = curr_data[(curr_data[SRC_IP].apply(is_ipv4)) | (curr_data[DST_IP].apply(is_ipv4))]\n",
    "    ipv6_addrs = curr_data[(curr_data[SRC_IP].apply(is_ipv6)) | (curr_data[DST_IP].apply(is_ipv6))]\n",
    "\n",
    "    return pd.concat([process_by_ipvx_range_pair(ipv4_addrs, ipv4_ranges_str, ipv4_in_range_cmp), \n",
    "                      process_by_ipvx_range_pair(ipv6_addrs, ipv6_ranges_str, ipv6_in_range_cmp)])\n",
    "\n",
    "def process_by_time_range(curr_data, time_period):\n",
    "    if curr_data.empty:\n",
    "        return curr_data\n",
    "    start_offset, end_offset = time_period\n",
    "\n",
    "    start = get_first_time_flow() + datetime.timedelta(minutes=start_offset)\n",
    "    end = get_first_time_flow() + datetime.timedelta(minutes=end_offset)\n",
    "\n",
    "    return curr_data[(curr_data[\"START TIME - FIRST SEEN\"].apply(to_datetime) >= start) &\n",
    "                      ((curr_data[\"START TIME - FIRST SEEN\"].apply(to_datetime)) <= end)]\n",
    "\n",
    "def process_by_dns_smtp(curr_data: pd.DataFrame, protocol_comm, top_n):\n",
    "    if protocol_comm == \"DNS_COMM\":\n",
    "        protocol_regex = \"dns\"\n",
    "    else:\n",
    "        protocol_regex = \"smtp\"\n",
    "\n",
    "    processed_data = curr_data[~curr_data[\"DNS QNAME\"].str.contains(protocol_regex, regex=True, na=False)]\n",
    "    return processed_data.groupby(SRC_IP).agg(COUNT=(SRC_IP, 'size'), PACKETS=('PACKETS', 'sum'), BYTES=('BYTES', 'sum')).reset_index().nlargest(top_n, columns=COUNT)\n",
    "\n",
    "def process_by_protocol(curr_data, protocols):\n",
    "    if curr_data.empty:\n",
    "        return curr_data\n",
    "    if \"ALL\" in protocols:\n",
    "        return curr_data\n",
    "\n",
    "    processed_data = pd.DataFrame()\n",
    "    for protocol in protocols:\n",
    "        if protocol in [\"TCP\", \"UDP\"]:\n",
    "            processed_data = pd.concat([processed_data, curr_data[curr_data[\"PROTOCOL\"] == protocol]])\n",
    "        else:\n",
    "            processed_data = pd.concat([processed_data, curr_data[curr_data[\"DETECTED PROTOCOL\"] == protocol]])\n",
    "\n",
    "    return processed_data\n",
    "\n",
    "def process_by_port(curr_data, ports, g_src_dst):\n",
    "    if curr_data.empty:\n",
    "        return curr_data\n",
    "\n",
    "    if ports == \"ALL\":\n",
    "        return curr_data\n",
    "\n",
    "    port_dir = \"SOURCE PORT\" if g_src_dst == SRC_IP else \"DESTINATION PORT\"\n",
    "\n",
    "    tmp_data = pd.DataFrame()\n",
    "    ports_list = ports.split(', ')\n",
    "\n",
    "    for port in ports_list:\n",
    "        tmp_data = pd.concat([tmp_data, curr_data[curr_data[port_dir] == int(port)]])\n",
    "\n",
    "    return tmp_data\n",
    "\n",
    "def get_traffic_data(g_src_dst, g_data_type, top_n, time_period,\n",
    "                     ipv4_range_str, ipv6_range_str, single_or_pair,\n",
    "                     protocol, ports, minimal_duration):\n",
    "    processed_data = pd.DataFrame(flow_data)\n",
    "\n",
    "    processed_data: pd.DataFrame = process_by_time_range(processed_data, time_period)\n",
    "    processed_data: pd.DataFrame = process_by_port(processed_data, ports, g_src_dst)\n",
    "    processed_data: pd.DataFrame = process_by_protocol(processed_data, protocol)\n",
    "    processed_data: pd.DataFrame = process_by_ip_range(processed_data, g_src_dst, ipv4_range_str.split(','), ipv6_range_str.split(','), single_or_pair)\n",
    "\n",
    "    processed_data: pd.DataFrame = process_by_type(processed_data, g_src_dst, g_data_type, top_n, single_or_pair, minimal_duration)\n",
    "    \n",
    "    processed_data.sort_values(processed_data.columns[1])\n",
    "        \n",
    "    return processed_data\n",
    "\n",
    "def on_change_single_or_pair(v):\n",
    "    if single_or_pair_widget.value == 'Pair':\n",
    "        disable_button(src_dst_widget)\n",
    "    else:\n",
    "        enable_button(src_dst_widget)\n",
    "\n",
    "def on_change_data_type(v):\n",
    "    clear_output(wait=True)\n",
    "    data_type = data_type_widget.value\n",
    "    \n",
    "    if data_type in [\"SMTP communication\", \"DNS communication\"]:\n",
    "        protocol_widget.disabled = True\n",
    "        protocol_widget.value = (\"ALL\",)\n",
    "    else:\n",
    "        protocol_widget.disabled = False\n",
    "\n",
    "    if data_type == 'Duration with dst IP':\n",
    "        minimal_duration_widget.disabled = False\n",
    "        minimal_duration_widget.value = \"5\"\n",
    "        disable_button(src_dst_widget)\n",
    "        single_or_pair_widget.value = 'Single'\n",
    "        disable_button(single_or_pair_widget)\n",
    "\n",
    "    elif data_type in ['IP requesting unusual TLD',\n",
    "                        'Subdomains of muni.cz',\n",
    "                        'Unusual TLDs', \n",
    "                        'Most DNS requests',\n",
    "                        'DNS communication',\n",
    "                        'SMTP communication']: \n",
    "        minimal_duration_widget.disabled = True\n",
    "        minimal_duration_widget.value = \"disabled\"\n",
    "        disable_button(src_dst_widget)\n",
    "        single_or_pair_widget.value = 'Single'\n",
    "        disable_button(single_or_pair_widget)\n",
    "    \n",
    "    elif single_or_pair_widget.value == 'Single' or single_or_pair_widget.disabled:\n",
    "            enable_button(src_dst_widget)\n",
    "            enable_button(single_or_pair_widget)\n",
    "    else: \n",
    "        disable_button(src_dst_widget)\n",
    "\n",
    "    if data_type != 'Duration with dst IP':\n",
    "        minimal_duration_widget.disabled = True\n",
    "        minimal_duration_widget.value = \"disabled\"\n",
    "\n",
    "\n",
    "def make_graph(g_data_type, top_n, graph_data):\n",
    "    if g_data_type in [\"DNS_COMM\", \"SMTP_COMM\"]:\n",
    "        tooltips = [\"COUNT\", \"BYTES\", \"PACKETS\"]\n",
    "    else:\n",
    "        tooltips = [g_data_type]  \n",
    "    if g_data_type in [\"BYTES\", \"PACKETS\", \"DURATION\", \n",
    "                       \"DNS QNAME\", \"UNUSUAL TLD\"]:\n",
    "        tooltips.append(\"COUNT\")\n",
    "\n",
    "    brush = alt.selection_interval(name=\"brush\")\n",
    "    graph = alt.Chart(graph_data).mark_bar().encode(\n",
    "        x=graph_data.columns[1],\n",
    "        y=alt.Y(graph_data.columns[0], sort='-x'),\n",
    "        tooltip=tooltips,\n",
    "        color=alt.Color(graph_data.columns[1],\n",
    "                   scale=alt.Scale(range=['lightgreen', 'green']))\n",
    "    ).properties(\n",
    "        width=600,\n",
    "        height=1000 if top_n >= 30 else 600,\n",
    "        autosize=alt.AutoSizeParams(\n",
    "            type='fit',\n",
    "            contains='padding'\n",
    "        ),\n",
    "    ).add_params(brush)\n",
    "    \n",
    "    return alt.JupyterChart(graph)\n",
    "    \n",
    "def on_select_ip_direction(change):\n",
    "    if src_dst_widget.value == \"Received\":\n",
    "        on_select_src_ip_address(change)\n",
    "    else:\n",
    "        on_select_dst_ip_address(change)\n",
    "\n",
    "def on_select_src_ip_address(change):\n",
    "    sel = change.new.value\n",
    "    if sel is None or SRC_IP not in sel:\n",
    "        filtered = flow_data.iloc[:0]\n",
    "    else:\n",
    "        filtered = flow_data[flow_data[SRC_IP].isin(sel[SRC_IP])]\n",
    "        filtered = process_by_port(filtered, port_widget.value, SRC_IP)\n",
    "        filtered = process_by_protocol(filtered, protocol_widget.value)\n",
    "        \n",
    "\n",
    "    table_widget.value = filtered.to_html()\n",
    "\n",
    "def on_select_dst_ip_address(change):\n",
    "    sel = change.new.value\n",
    "    if sel is None or DST_IP not in sel:\n",
    "        filtered = flow_data.iloc[:0]\n",
    "    else:\n",
    "        filtered = flow_data[flow_data[DST_IP].isin(sel[DST_IP])]\n",
    "        filtered = process_by_port(filtered, port_widget.value, DST_IP)\n",
    "        filtered = process_by_protocol(filtered, protocol_widget.value)\n",
    "\n",
    "    table_widget.value = filtered.to_html()\n",
    "\n",
    "def on_select_src_and_dst_ip_address(change):\n",
    "    sel = change.new.value\n",
    "    if sel is None or \"pair\" not in sel:\n",
    "        ret_df = flow_data.iloc[:0]\n",
    "    else:\n",
    "        ret_df = pd.DataFrame()\n",
    "\n",
    "        for i in range(len(sel[\"pair\"])):\n",
    "            filtered = flow_data[((flow_data[SRC_IP] == sel[\"pair\"][i][0]) & (flow_data[DST_IP] == sel[\"pair\"][i][1]))  |\n",
    "                            (flow_data[DST_IP] == sel[\"pair\"][i][0]) & (flow_data[SRC_IP] == sel[\"pair\"][i][1])]\n",
    "            ret_df = pd.concat([ret_df, filtered])\n",
    "\n",
    "        ret_df = process_by_port(ret_df, port_widget.value, DST_IP)\n",
    "        ret_df = process_by_protocol(ret_df, protocol_widget.value)\n",
    "\n",
    "    table_widget.value = ret_df.to_html()\n",
    "\n",
    "def on_select_unusual_tld(change):\n",
    "    sel = change.new.value\n",
    "    if sel is None or \"UNUSUAL TLD\" not in sel:\n",
    "        filtered = flow_data.iloc[:0]\n",
    "    else:\n",
    "        filtered = flow_data[flow_data[\"DNS QNAME\"].isin(sel[\"UNUSUAL TLD\"])]\n",
    "        filtered = process_by_port(filtered, port_widget.value, DST_IP)\n",
    "        filtered = process_by_protocol(filtered, protocol_widget.value)\n",
    "\n",
    "    table_widget.value = filtered.to_html()\n",
    "\n",
    "def on_select_muni_subdomain(change):\n",
    "    sel = change.new.value\n",
    "    if sel is None or \"DNS QNAME\" not in sel:\n",
    "        filtered = flow_data.iloc[:0]\n",
    "    else:\n",
    "        filtered = flow_data[flow_data[\"DNS QNAME\"].isin(sel[\"DNS QNAME\"])]\n",
    "        filtered = process_by_port(filtered, port_widget.value, DST_IP)\n",
    "        filtered = process_by_protocol(filtered, protocol_widget.value)\n",
    "\n",
    "    table_widget.value = filtered.to_html()\n",
    "\n",
    "def on_change(v):\n",
    "\n",
    "    clear_output(wait=True)\n",
    "    display_widgets()\n",
    "\n",
    "    on_select_func = None\n",
    "    data_type_key = data_type_widget.value\n",
    "    single_pair_key = single_or_pair_widget.value\n",
    "\n",
    "    g_src_dst = SRC_IP if src_dst_widget.value == \"Received\" else DST_IP\n",
    "    \n",
    "    if (DATA_TYPE_DSC[data_type_key] in [\"BYTES\", \"PACKETS\", \n",
    "                                         \"FLOWS\", \"IP REQUESTING UNUSUAL TLD\",\n",
    "                                         \"MOST DNS REQUESTS\", \"DNS_COMM\", \"SMTP_COMM\"] \n",
    "        and SINGLE_PAIR_DSC[single_pair_key] == \"SINGLE\"):\n",
    "        on_select_func = on_select_ip_direction\n",
    "    elif DATA_TYPE_DSC[data_type_key] == \"UNUSUAL TLD\":\n",
    "        on_select_func = on_select_unusual_tld\n",
    "    elif DATA_TYPE_DSC[data_type_key] == \"DURATION\":\n",
    "        on_select_func = on_select_dst_ip_address\n",
    "    elif DATA_TYPE_DSC[data_type_key] == \"DNS QNAME\":\n",
    "        on_select_func = on_select_muni_subdomain\n",
    "    elif SINGLE_PAIR_DSC[single_pair_key] == \"PAIR\":\n",
    "        on_select_func = on_select_src_and_dst_ip_address\n",
    "\n",
    "    graph_data = get_traffic_data(g_src_dst, DATA_TYPE_DSC[data_type_key],\n",
    "                            int(top_n_widget.value), period_widget.value,\n",
    "                            ipv4_range_widget.value,ipv6_range_widget.value,\n",
    "                            single_or_pair_widget.value, protocol_widget.value,\n",
    "                            port_widget.value, minimal_duration_widget.value)\n",
    "\n",
    "    if graph_data.empty:\n",
    "        print(\"Log: The data set is empty\")\n",
    "        return\n",
    "    graph: alt.JupyterChart = make_graph(DATA_TYPE_DSC[data_type_key], \n",
    "                                         int(top_n_widget.value), \n",
    "                                         graph_data)\n",
    "\n",
    "    if (on_select_func):\n",
    "        graph.selections.observe(on_select_func, [\"brush\"])\n",
    "\n",
    "    display(widgets.VBox([graph, table_widget]))\n",
    "\n",
    "\n",
    "single_or_pair_widget.on_trait_change(on_change_single_or_pair)\n",
    "data_type_widget.on_trait_change(on_change_data_type)\n",
    "apply_changes_widget.on_click(on_change)\n",
    "display_widgets()\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}