ISAA_Review_3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ISAA_Review_3.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7sO5HF8GaQsF"
      },
      "source": [
        "## Dataset Install"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4NZ3qtaQMwwy"
      },
      "source": [
        "🚀 Dataset Install:\n",
        "<ol> \n",
        "  <li> AWS and Sudo installed in the runtime </li>\n",
        "  <li> Using Sudo, AWS commands are activated </li>\n",
        "  <li> CIC IDS 2018 installed from AWS bucket </li>\n",
        "</ol> "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3nJGHQ6vJWSK"
      },
      "source": [
        "import time"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lyC4cqdKaKBj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d0941ec1-87cf-4c34-df2b-e118f27522f1"
      },
      "source": [
        "#Package to hide cell outputs. Cells to be hidden are decorated with %%capture\n",
        "!pip install -q gwpy"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.4 MB 5.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 11.2 MB 33.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 51 kB 5.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 55 kB 2.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.6 MB 57.3 MB/s \n",
            "\u001b[?25h  Building wheel for ligo-segments (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3EvQ1T6WAZv"
      },
      "source": [
        "%%capture\n",
        "!apt-get install sudo -y"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JsFVqTwKXFHV"
      },
      "source": [
        "%%capture\n",
        "!curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\"\n",
        "!unzip awscliv2.zip"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H6icg72bWbrz",
        "outputId": "fe6275b2-f893-4cbe-c6ed-765887bb8573"
      },
      "source": [
        "!sudo ./aws/install"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "You can now run: /usr/local/bin/aws --version\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PaJ5isXBXwP4"
      },
      "source": [
        "%%capture\n",
        "!mkdir cicids2018\n",
        "!aws s3 cp \"s3://cse-cic-ids2018/Processed Traffic Data for ML Algorithms/\" ./cicids2018  --recursive --no-sign-request"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "11NmrhD9YVM3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9eabe1f3-f4b0-430e-c395-4b8c6d159b86"
      },
      "source": [
        "!ls cicids2018"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Friday-02-03-2018_TrafficForML_CICFlowMeter.csv\n",
            "Friday-16-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Friday-23-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Thuesday-20-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Thursday-01-03-2018_TrafficForML_CICFlowMeter.csv\n",
            "Thursday-15-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Thursday-22-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Wednesday-14-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Wednesday-21-02-2018_TrafficForML_CICFlowMeter.csv\n",
            "Wednesday-28-02-2018_TrafficForML_CICFlowMeter.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PuWERTcKs8yk",
        "cellView": "form"
      },
      "source": [
        "#@title\n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import plotly.express as px \n",
        "\n",
        "# df = pd.read_csv(\"cicids2018/Wednesday-14-02-2018_TrafficForML_CICFlowMeter.csv\")\n",
        "# value_counts = df['Label'].value_counts().reset_index()\n",
        "# fig = px.pie(value_counts, values='Label', names='index', color='index',\n",
        "#              title='Label Counts in the Dataset',\n",
        "#              color_discrete_map={'Benign':'lightcyan',\n",
        "#                                  'FTP-BruteForce':'cyan',\n",
        "#                                  'SSH-BruteForce':'royalblue'},\n",
        "#              width = 500,\n",
        "#              height = 500)\n",
        "# # fig.show()\n",
        "\n",
        "# df['Flow Byts/s'].fillna(df['Flow Byts/s'].median(),inplace = True)\n",
        "# df.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
        "# df['Flow Pkts/s'].fillna(400000, inplace = True)\n",
        "# df['Label'] = df['Label'].map({\n",
        "#     'Benign' : 0,\n",
        "#     'FTP-BruteForce'  :1,\n",
        "#     'SSH-BruteForce' : 2\n",
        "# })\n",
        "# df['Label'].fillna(2,inplace = True)\n",
        "\n",
        "# px.imshow(df.corr(),title = \"Correlation between features \", width = 500,height = 500)\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mmqrCYbkafE3"
      },
      "source": [
        "## Pyspark Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WSvXMgyyMcP0"
      },
      "source": [
        "🚀 Pyspark Setup:\n",
        "<ol> \n",
        "  <li> Java and spark installed </li>\n",
        "  <li> Env Variables for the same are set </li>\n",
        "  <li> Using findspark package initialise spark session </li>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8orIr2_LajFg"
      },
      "source": [
        "%%capture\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://mirrors.estointernet.in/apache/spark/spark-3.1.2/spark-3.1.2-bin-hadoop2.7.tgz\n",
        "!tar xf spark-3.1.2-bin-hadoop2.7.tgz"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6HY0ks3annd"
      },
      "source": [
        "import os \n",
        "os.environ[\"JAVA_HOME\"] =  \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.2-bin-hadoop2.7\""
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePRyDR-xave2"
      },
      "source": [
        "!pip install -q findspark\n",
        "import findspark\n",
        "findspark.init()"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 239
        },
        "id": "RlCueVKmayPr",
        "outputId": "35a82d66-db12-489f-f89b-b8faa2dee790"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder\\\n",
        "        .master(\"local[*]\")\\\n",
        "        .appName(\"ISAA_CIC_IDS_2018\")\\\n",
        "        .config('spark.ui.port', '4050')\\\n",
        "        .getOrCreate()\n",
        "print(\"Spark Session Created\")\n",
        "spark"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark Session Created\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://2e347e5b2c23:4050\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>ISAA_CIC_IDS_2018</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f01e452cd50>"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nT6oRUs-tx68"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from pyspark.sql.functions import col, lit, udf, when,count,isnan\n",
        "from pyspark.sql.types import DoubleType\n",
        "\n",
        "from pyspark.sql.functions import max as sparkMax\n",
        "from pyspark.sql.functions import min as sparkMin\n",
        "from pyspark.sql.functions import first, col\n",
        "\n",
        "from pyspark.ml.feature import Imputer"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wXQgqxQLa_7W"
      },
      "source": [
        "## Dataset Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WiAI-I6_Ky36"
      },
      "source": [
        "🚀 Dataset preprocessing:\n",
        "<ol>\n",
        "  <li> Dataset read and checked for Null and Infinity Values</li>\n",
        "  <li> Infinity Values replaced with the max value in the column </li>\n",
        "  <li> Null Values in columns handled using Imputer using median of column</li>\n",
        "  <li> Categorical Column which includes the labels then encoded to numbers </li>\n",
        "</ol>\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FSfxQ0EYbJZT",
        "outputId": "340921e6-4c8a-463e-e305-904cc51e43a7"
      },
      "source": [
        "spark_df = spark.read.csv(\"cicids2018/Wednesday-14-02-2018_TrafficForML_CICFlowMeter.csv\",header  =True, inferSchema = True)\n",
        "spark_df.show(2)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+------------+-------------+--------------+------------+------------+-----------+------------+--------------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+------------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+\n",
            "|Dst Port|Protocol|          Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|Flow Byts/s| Flow Pkts/s|Flow IAT Mean|  Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|Fwd IAT Mean|   Fwd IAT Std|Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|Bwd IAT Mean|Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|  Fwd Pkts/s|Bwd Pkts/s|Pkt Len Min|Pkt Len Max|Pkt Len Mean|Pkt Len Std|Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|   Idle Mean|      Idle Std|Idle Max|Idle Min| Label|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+------------+-------------+--------------+------------+------------+-----------+------------+--------------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+------------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+\n",
            "|       0|       0|14/02/2018 08:31:01|    112641719|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|        0.0|0.0266331163| 5.63208595E7|139.3000358938|    56320958|    56320761|  112641719|5.63208595E7|139.3000358938|   56320958|   56320761|          0|         0.0|        0.0|          0|          0|            0|            0|            0|            0|             0|             0|0.0266331163|       0.0|          0|          0|         0.0|        0.0|        0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|         0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0|5.63208595E7|139.3000358938|56320958|56320761|Benign|\n",
            "|       0|       0|14/02/2018 08:33:50|    112641466|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|        0.0|0.0266331761|  5.6320733E7|114.5512985522|    56320814|    56320652|  112641466| 5.6320733E7|114.5512985522|   56320814|   56320652|          0|         0.0|        0.0|          0|          0|            0|            0|            0|            0|             0|             0|0.0266331761|       0.0|          0|          0|         0.0|        0.0|        0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|         0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0| 5.6320733E7|114.5512985522|56320814|56320652|Benign|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+------------+-------------+--------------+------------+------------+-----------+------------+--------------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+------------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uRZNNLFnjaOD",
        "outputId": "69e2c693-cd94-4a37-a002-31ae277edc62"
      },
      "source": [
        "spark_df.printSchema()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Dst Port: integer (nullable = true)\n",
            " |-- Protocol: integer (nullable = true)\n",
            " |-- Timestamp: string (nullable = true)\n",
            " |-- Flow Duration: long (nullable = true)\n",
            " |-- Tot Fwd Pkts: integer (nullable = true)\n",
            " |-- Tot Bwd Pkts: integer (nullable = true)\n",
            " |-- TotLen Fwd Pkts: integer (nullable = true)\n",
            " |-- TotLen Bwd Pkts: integer (nullable = true)\n",
            " |-- Fwd Pkt Len Max: integer (nullable = true)\n",
            " |-- Fwd Pkt Len Min: integer (nullable = true)\n",
            " |-- Fwd Pkt Len Mean: double (nullable = true)\n",
            " |-- Fwd Pkt Len Std: double (nullable = true)\n",
            " |-- Bwd Pkt Len Max: integer (nullable = true)\n",
            " |-- Bwd Pkt Len Min: integer (nullable = true)\n",
            " |-- Bwd Pkt Len Mean: double (nullable = true)\n",
            " |-- Bwd Pkt Len Std: double (nullable = true)\n",
            " |-- Flow Byts/s: double (nullable = true)\n",
            " |-- Flow Pkts/s: double (nullable = true)\n",
            " |-- Flow IAT Mean: double (nullable = true)\n",
            " |-- Flow IAT Std: double (nullable = true)\n",
            " |-- Flow IAT Max: long (nullable = true)\n",
            " |-- Flow IAT Min: long (nullable = true)\n",
            " |-- Fwd IAT Tot: long (nullable = true)\n",
            " |-- Fwd IAT Mean: double (nullable = true)\n",
            " |-- Fwd IAT Std: double (nullable = true)\n",
            " |-- Fwd IAT Max: long (nullable = true)\n",
            " |-- Fwd IAT Min: long (nullable = true)\n",
            " |-- Bwd IAT Tot: integer (nullable = true)\n",
            " |-- Bwd IAT Mean: double (nullable = true)\n",
            " |-- Bwd IAT Std: double (nullable = true)\n",
            " |-- Bwd IAT Max: integer (nullable = true)\n",
            " |-- Bwd IAT Min: integer (nullable = true)\n",
            " |-- Fwd PSH Flags: integer (nullable = true)\n",
            " |-- Bwd PSH Flags: integer (nullable = true)\n",
            " |-- Fwd URG Flags: integer (nullable = true)\n",
            " |-- Bwd URG Flags: integer (nullable = true)\n",
            " |-- Fwd Header Len: integer (nullable = true)\n",
            " |-- Bwd Header Len: integer (nullable = true)\n",
            " |-- Fwd Pkts/s: double (nullable = true)\n",
            " |-- Bwd Pkts/s: double (nullable = true)\n",
            " |-- Pkt Len Min: integer (nullable = true)\n",
            " |-- Pkt Len Max: integer (nullable = true)\n",
            " |-- Pkt Len Mean: double (nullable = true)\n",
            " |-- Pkt Len Std: double (nullable = true)\n",
            " |-- Pkt Len Var: double (nullable = true)\n",
            " |-- FIN Flag Cnt: integer (nullable = true)\n",
            " |-- SYN Flag Cnt: integer (nullable = true)\n",
            " |-- RST Flag Cnt: integer (nullable = true)\n",
            " |-- PSH Flag Cnt: integer (nullable = true)\n",
            " |-- ACK Flag Cnt: integer (nullable = true)\n",
            " |-- URG Flag Cnt: integer (nullable = true)\n",
            " |-- CWE Flag Count: integer (nullable = true)\n",
            " |-- ECE Flag Cnt: integer (nullable = true)\n",
            " |-- Down/Up Ratio: integer (nullable = true)\n",
            " |-- Pkt Size Avg: double (nullable = true)\n",
            " |-- Fwd Seg Size Avg: double (nullable = true)\n",
            " |-- Bwd Seg Size Avg: double (nullable = true)\n",
            " |-- Fwd Byts/b Avg: integer (nullable = true)\n",
            " |-- Fwd Pkts/b Avg: integer (nullable = true)\n",
            " |-- Fwd Blk Rate Avg: integer (nullable = true)\n",
            " |-- Bwd Byts/b Avg: integer (nullable = true)\n",
            " |-- Bwd Pkts/b Avg: integer (nullable = true)\n",
            " |-- Bwd Blk Rate Avg: integer (nullable = true)\n",
            " |-- Subflow Fwd Pkts: integer (nullable = true)\n",
            " |-- Subflow Fwd Byts: integer (nullable = true)\n",
            " |-- Subflow Bwd Pkts: integer (nullable = true)\n",
            " |-- Subflow Bwd Byts: integer (nullable = true)\n",
            " |-- Init Fwd Win Byts: integer (nullable = true)\n",
            " |-- Init Bwd Win Byts: integer (nullable = true)\n",
            " |-- Fwd Act Data Pkts: integer (nullable = true)\n",
            " |-- Fwd Seg Size Min: integer (nullable = true)\n",
            " |-- Active Mean: double (nullable = true)\n",
            " |-- Active Std: double (nullable = true)\n",
            " |-- Active Max: integer (nullable = true)\n",
            " |-- Active Min: integer (nullable = true)\n",
            " |-- Idle Mean: double (nullable = true)\n",
            " |-- Idle Std: double (nullable = true)\n",
            " |-- Idle Max: long (nullable = true)\n",
            " |-- Idle Min: long (nullable = true)\n",
            " |-- Label: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1nryMNjalhAH",
        "outputId": "2bfa0483-0eb7-41a1-dbe7-da369f6257b0"
      },
      "source": [
        "#Checking for Null Values\n",
        "spark_df.select([count(when(isnan(c), c)).alias(c) for c in spark_df.columns]).show()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "|Dst Port|Protocol|Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|Flow Byts/s|Flow Pkts/s|Flow IAT Mean|Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|Fwd IAT Mean|Fwd IAT Std|Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|Bwd IAT Mean|Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|Fwd Pkts/s|Bwd Pkts/s|Pkt Len Min|Pkt Len Max|Pkt Len Mean|Pkt Len Std|Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|Idle Mean|Idle Std|Idle Max|Idle Min|Label|\n",
            "+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "|       0|       0|        0|            0|           0|           0|              0|              0|              0|              0|               0|              0|              0|              0|               0|              0|       2277|          0|            0|           0|           0|           0|          0|           0|          0|          0|          0|          0|           0|          0|          0|          0|            0|            0|            0|            0|             0|             0|         0|         0|          0|          0|           0|          0|          0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0|               0|               0|             0|             0|               0|             0|             0|               0|               0|               0|               0|               0|                0|                0|                0|               0|          0|         0|         0|         0|        0|       0|       0|       0|    0|\n",
            "+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dc9FD-mkoWMf"
      },
      "source": [
        "seq_of_columns = spark_df.columns\n",
        "\n",
        "# Using List comprehensions to create a list of columns of String DataType\n",
        "string_columns = [i[0] for i in spark_df.dtypes if i[1]=='string']\n",
        "\n",
        "# Using Set function to get non-string columns by subtracting one list from another.\n",
        "non_string_columns = list(set(seq_of_columns) - set(string_columns))"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUnpsyvIofCY",
        "outputId": "d6d2d951-f1fd-4867-86d1-2c92dd061864"
      },
      "source": [
        "print(\"Checking for the presence of infinite values\")\n",
        "df = spark_df.select(*[sparkMax(col(c)).alias(c) for c in non_string_columns],*[first(col(c),ignorenulls = True).alias(c) for c in string_columns])\n",
        "df = df[[seq_of_columns]]\n",
        "df.show()\n",
        "print(\"Flow Pkts/s contains infinity\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Checking for the presence of infinite values\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+----------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+-------------------+------------+------------+-----------+------------+-------------------+------------+-----------+-----------+------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+---------------+-------------+-----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------+------------------+----------+----------+--------------+-------------------+------------+-----------+------+\n",
            "|Dst Port|Protocol|          Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean| Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|Flow Byts/s|Flow Pkts/s|Flow IAT Mean|       Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|Fwd IAT Mean|        Fwd IAT Std| Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|Bwd IAT Mean|     Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|Fwd Pkts/s|Bwd Pkts/s|Pkt Len Min|Pkt Len Max|   Pkt Len Mean|  Pkt Len Std|      Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|  Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min| Active Mean|        Active Std|Active Max|Active Min|     Idle Mean|           Idle Std|    Idle Max|   Idle Min| Label|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+----------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+-------------------+------------+------------+-----------+------------+-------------------+------------+-----------+-----------+------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+---------------+-------------+-----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------+------------------+----------+----------+--------------+-------------------+------------+-----------+------+\n",
            "|   65533|      17|14/02/2018 08:31:01|    119999989|        5115|        9198|        8591554|       13397730|          64440|           1460|11217.0252252252|15760.8375895074|           1460|            732|  1459.240494732| 944.6946596652|        NaN|   Infinity| 1.19990721E8|4.24473900119579E11|979781000000|   119990721|  119999849|1.19990721E8|4.24473900119579E11|979781000000|  119990721|  119999719|1.19400326E8|8.447314760547E7|  119564005|  119400326|            1|            0|            0|            0|        157688|        183972| 4000000.0| 2000000.0|        788|      64440|2521.1157426143|8810.76844946|7.7629640669999E7|           1|           1|           1|           1|           1|           1|             0|           1|           80|2522.136437247|11217.0252252252|  1459.240494732|             0|             0|               0|             0|             0|               0|            5115|         8591554|            9198|        13397730|            65535|            65535|             1031|              48|1.10240118E8|5.72344617203198E7| 110240118| 110240118|3.394503125E11|2.43268209256922E11|979781000000|12603000000|Benign|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+----------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+-------------------+------------+------------+-----------+------------+-------------------+------------+-----------+-----------+------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+---------------+-------------+-----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------+------------------+----------+----------+--------------+-------------------+------------+-----------+------+\n",
            "\n",
            "Flow Pkts/s contains infinity\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k0yUPD9ot1pd",
        "outputId": "62bdee6b-1bc1-4315-8ea1-353c85826ddf"
      },
      "source": [
        "print(\"Checking for the presence of negative infinite values\")\n",
        "df = spark_df.select(*[sparkMin(col(c)).alias(c) for c in non_string_columns],*[first(col(c),ignorenulls = True).alias(c) for c in string_columns])\n",
        "df = df[[seq_of_columns]]\n",
        "df.show() \n",
        "print(\"No column contains negative infinity\")"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Checking for the presence of negative infinite values\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-------------+-------------+------------+-------------+-------------+-------------+------------+-----------+-------------+-------------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+------+\n",
            "|Dst Port|Protocol|          Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|Flow Byts/s|  Flow Pkts/s|Flow IAT Mean|Flow IAT Std| Flow IAT Max| Flow IAT Min|  Fwd IAT Tot|Fwd IAT Mean|Fwd IAT Std|  Fwd IAT Max|  Fwd IAT Min|Bwd IAT Tot|Bwd IAT Mean|Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|Fwd Pkts/s|Bwd Pkts/s|Pkt Len Min|Pkt Len Max|Pkt Len Mean|Pkt Len Std|Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|Idle Mean|Idle Std|Idle Max|Idle Min| Label|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-------------+-------------+------------+-------------+-------------+-------------+------------+-----------+-------------+-------------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+------+\n",
            "|       0|       0|14/02/2018 08:31:01|-919011000000|           1|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|        0.0|-0.0010297608|  -3.40701E11|         0.0|-159438000000|-943564000000|-919011000000| -3.40701E11|        0.0|-159438000000|-943564000000|          0|         0.0|        0.0|          0|          0|            0|            0|            0|            0|             0|             0|       0.0|       0.0|          0|          0|         0.0|        0.0|        0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|         0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               1|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0|      0.0|     0.0|       0|       0|Benign|\n",
            "+--------+--------+-------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-------------+-------------+------------+-------------+-------------+-------------+------------+-----------+-------------+-------------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+------+\n",
            "\n",
            "No column contains negative infinity\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F0UAYABswRk9",
        "outputId": "8b339a42-98c2-4a48-e32d-4323fce7c632"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5LbFIrp6ZuwJ"
      },
      "source": [
        "replace_infs_udf = udf(\n",
        "    lambda x, v: float(v) if x and np.isinf(x) else x, DoubleType()\n",
        ")\n",
        "temp = spark_df.withColumn(\"Flow\", replace_infs_udf(col(\"Flow Pkts/s\"), lit(-100)))"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fnhpgTFawumo",
        "outputId": "74e06ce3-4579-4f0d-a3c5-af924364d47c"
      },
      "source": [
        "des = temp.agg({\"Flow\":\"max\"}).collect()[0]\n",
        "des"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Row(max(Flow)=4000000.0)"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "By7ki9dUvetM",
        "outputId": "874f6a35-045d-4b74-e01c-77199df33d4d"
      },
      "source": [
        "spark_df = spark_df.withColumn(\"Flow Pkts/s\", replace_infs_udf(col(\"Flow Pkts/s\"), lit(4000000.0)))\n",
        "spark_df.agg({\"Flow Pkts/s\":\"max\"}).collect()[0]\n",
        "print(\"Infinite Values replaced\")"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Infinite Values replaced\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrlq6TOxyZLM"
      },
      "source": [
        "columns_to_drop = ['Timestamp']\n",
        "spark_df = spark_df.drop(*columns_to_drop)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnDdBKuey726"
      },
      "source": [
        "imputer = Imputer(inputCol=\"Flow Byts/s\", outputCol=\"Flow Byts/s\",strategy = \"median\")\n",
        "imputer_mod = imputer.fit(spark_df)\n",
        "spark_df = imputer_mod.transform(spark_df)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0US8FrKCYral"
      },
      "source": [
        "## Modelling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9ZZpS_YNJAI"
      },
      "source": [
        "🚀 Modelling:\n",
        "<ol>\n",
        "<li> All independent features compiled using vector assembler </li> \n",
        "<li> The data is scaled in the range of 0 and 1 </li>\n",
        "<li> Data split into train-test in 80/20 ratio</li>\n",
        "<li> Random Forest, Decision Tree and Naive Bayes classification models trained </li>\n",
        "<li> Test predictions evaluated using Accuracy </li>\n",
        "</ol>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pxnhqgv9YdPW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56f62a8f-cea7-427d-e311-961b846e1d47"
      },
      "source": [
        "from pyspark.ml.feature import VectorAssembler \n",
        "feature_assembler = VectorAssembler(inputCols = spark_df.columns[:-1],\n",
        "                                    outputCol = \"independent_features\")\n",
        "training = feature_assembler.transform(spark_df)\n",
        "training.show()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+---------------+-------------+----------------+----------------+------------+------------+-----------+----------------+----------------+-----------+-----------+-----------+----------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------+------------+-----------+-----------+--------------+--------------+----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+--------------------+\n",
            "|Dst Port|Protocol|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|    Flow Byts/s|  Flow Pkts/s|   Flow IAT Mean|    Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|    Fwd IAT Mean|     Fwd IAT Std|Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|    Bwd IAT Mean|     Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|   Fwd Pkts/s|  Bwd Pkts/s|Pkt Len Min|Pkt Len Max|  Pkt Len Mean|   Pkt Len Std|     Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|  Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|   Idle Mean|      Idle Std|Idle Max|Idle Min| Label|independent_features|\n",
            "+--------+--------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+---------------+-------------+----------------+----------------+------------+------------+-----------+----------------+----------------+-----------+-----------+-----------+----------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------+------------+-----------+-----------+--------------+--------------+----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+--------------------+\n",
            "|       0|       0|    112641719|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|            0.0| 0.0266331163|    5.63208595E7|  139.3000358938|    56320958|    56320761|  112641719|    5.63208595E7|  139.3000358938|   56320958|   56320761|          0|             0.0|             0.0|          0|          0|            0|            0|            0|            0|             0|             0| 0.0266331163|         0.0|          0|          0|           0.0|           0.0|             0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0|5.63208595E7|139.3000358938|56320958|56320761|Benign|(78,[2,3,16,17,18...|\n",
            "|       0|       0|    112641466|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|            0.0| 0.0266331761|     5.6320733E7|  114.5512985522|    56320814|    56320652|  112641466|     5.6320733E7|  114.5512985522|   56320814|   56320652|          0|             0.0|             0.0|          0|          0|            0|            0|            0|            0|             0|             0| 0.0266331761|         0.0|          0|          0|           0.0|           0.0|             0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0| 5.6320733E7|114.5512985522|56320814|56320652|Benign|(78,[2,3,16,17,18...|\n",
            "|       0|       0|    112638623|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|            0.0| 0.0266338483|    5.63193115E7|  301.9345955667|    56319525|    56319098|  112638623|    5.63193115E7|  301.9345955667|   56319525|   56319098|          0|             0.0|             0.0|          0|          0|            0|            0|            0|            0|             0|             0| 0.0266338483|         0.0|          0|          0|           0.0|           0.0|             0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0|5.63193115E7|301.9345955667|56319525|56319098|Benign|(78,[2,3,16,17,18...|\n",
            "|      22|       6|      6453966|          15|          10|           1239|           2273|            744|              0|            82.6| 196.7412368715|            976|              0|           227.3| 371.6778922072| 544.1615279659| 3.8735871865|       268915.25|247443.778966007|      673900|          22|    6453966|460997.571428571|123109.423587757|     673900|     229740|    5637902|626433.555555556| 455082.21422401|    1167293|        554|            0|            0|            0|            0|           488|           328| 2.3241523119|1.5494348746|          0|        976|135.0769230769|277.8347599674|77192.1538461539|           0|           0|           0|           1|           0|           0|             0|           0|            0|        140.48|            82.6|           227.3|             0|             0|               0|             0|             0|               0|              15|            1239|              10|            2273|            65535|              233|                6|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      22|       6|      8804066|          14|          11|           1143|           2209|            744|              0|   81.6428571429| 203.7455453568|            976|              0|  200.8181818182| 362.2498635422| 380.7331748762|  2.839597068|366836.083333333|511356.609732762|     1928102|          21|    8804066|677235.846153846|532416.970958985|    1928102|     246924|    7715481|        771548.1|755543.082716951|    2174893|         90|            0|            0|            0|            0|           456|           360| 1.5901743581|1.2494227099|          0|        976|128.9230769231|279.7630315931|78267.3538461539|           0|           0|           0|           1|           0|           0|             0|           0|            0|        134.08|   81.6428571429|  200.8181818182|             0|             0|               0|             0|             0|               0|              14|            1143|              11|            2209|             5808|              233|                6|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      22|       6|      6989341|          16|          12|           1239|           2273|            744|              0|         77.4375| 190.8311535538|            976|              0|  189.4166666667| 347.6425694023| 502.4794183028| 4.0061001459|258864.481481481| 291724.14791076|      951098|          20|    6989341|465956.066666667|244363.896416351|     951098|     265831|    5980598|543690.727272727|460713.519752371|    1254338|         78|            0|            0|            0|            0|           332|           252| 2.2892000834|1.7169000625|          0|        976|121.1034482759|265.7086676402|70601.0960591133|           0|           0|           0|           1|           0|           0|             0|           0|            0|125.4285714286|         77.4375|  189.4166666667|             0|             0|               0|             0|             0|               0|              16|            1239|              12|            2273|             5808|              234|                7|              20|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|       0|       0|    112640480|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|            0.0| 0.0266334092|      5.632024E7|  203.6467529817|    56320384|    56320096|  112640480|      5.632024E7|  203.6467529817|   56320384|   56320096|          0|             0.0|             0.0|          0|          0|            0|            0|            0|            0|             0|             0| 0.0266334092|         0.0|          0|          0|           0.0|           0.0|             0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0|  5.632024E7|203.6467529817|56320384|56320096|Benign|(78,[2,3,16,17,18...|\n",
            "|       0|       0|    112641244|           3|           0|              0|              0|              0|              0|             0.0|            0.0|              0|              0|             0.0|            0.0|            0.0| 0.0266332286|     5.6320622E7|   62.2253967444|    56320666|    56320578|  112641244|     5.6320622E7|   62.2253967444|   56320666|   56320578|          0|             0.0|             0.0|          0|          0|            0|            0|            0|            0|             0|             0| 0.0266332286|         0.0|          0|          0|           0.0|           0.0|             0.0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0.0|             0.0|             0.0|             0|             0|               0|             0|             0|               0|               3|               0|               0|               0|               -1|               -1|                0|               0|        0.0|       0.0|         0|         0| 5.6320622E7| 62.2253967444|56320666|56320578|Benign|(78,[2,3,16,17,18...|\n",
            "|      80|       6|       476513|           5|           3|            211|            463|            211|              0|            42.2|  94.3620686505|            463|              0|  154.3333333333| 267.3131746348|1414.4419984345|16.7886290615|68073.2857142857|115865.792656438|      237711|          24|     476513|       119128.25|137379.963358017|     238470|        108|     238634|        119317.0|167621.076693833|     237843|        791|            0|            0|            0|            0|           168|           104|10.4928931635|6.2957358981|          0|        463| 74.8888888889|161.4058893322|26051.8611111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|         84.25|            42.2|  154.3333333333|             0|             0|               0|             0|             0|               0|               5|             211|               3|             463|            14600|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       475048|           5|           3|            220|            472|            220|              0|            44.0|    98.38699101|            472|              0|  157.3333333333| 272.5093270575|1456.6949024099|16.8404034961|         67864.0|115746.933154476|      237494|          15|     475048|        118762.0|137096.759626185|     237853|         15|     237516|        118758.0|167472.584269784|     237179|        337|            0|            0|            0|            0|           168|           104| 10.525252185| 6.315151311|          0|        472| 76.8888888889|165.0669897681|27247.1111111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|          86.5|            44.0|  157.3333333333|             0|             0|               0|             0|             0|               0|               5|             220|               3|             472|            14600|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       474926|           5|           3|            220|            472|            220|              0|            44.0|    98.38699101|            472|              0|  157.3333333333| 272.5093270575|1457.0691012916|16.8447294947|67846.5714285714|115645.740842248|      237162|          15|     474926|        118731.5|136923.365842601|     237497|         15|     237732|        118866.0|167663.503100705|     237422|        310|            0|            0|            0|            0|           168|           104|10.5279559342|6.3167735605|          0|        472| 76.8888888889|165.0669897681|27247.1111111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|          86.5|            44.0|  157.3333333333|             0|             0|               0|             0|             0|               0|               5|             220|               3|             472|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       477471|           5|           3|            209|            461|            209|              0|            41.8|  93.4676414595|            461|              0|  153.6666666667| 266.1584740964|1403.2265833946|16.7549442793|68210.1428571429|116178.228792989|      238389|          17|     477471|       119367.75|137516.508224467|     238515|        149|     238887|        119443.5|168454.755588852|     238559|        328|            0|            0|            0|            0|           168|           104|10.4718401746|6.2831041048|          0|        461| 74.4444444444|160.5942955954|25790.5277777778|           0|           0|           0|           1|           0|           0|             0|           0|            0|         83.75|            41.8|  153.6666666667|             0|             0|               0|             0|             0|               0|               5|             209|               3|             461|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       512758|           5|           3|            211|            463|            211|              0|            42.2|  94.3620686505|            463|              0|  154.3333333333| 267.3131746348|1314.4602327024|15.6019018718|73251.1428571429|124959.473740394|      256188|          10|     512758|        128189.5|148006.106082373|     256523|         10|     256563|        128281.5|180935.190276795|     256222|        341|            0|            0|            0|            0|           168|           104| 9.7511886699|5.8507132019|          0|        463| 74.8888888889|161.4058893322|26051.8611111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|         84.25|            42.2|  154.3333333333|             0|             0|               0|             0|             0|               0|               5|             211|               3|             463|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       476711|           5|           3|            206|            458|            206|              0|            41.2|   92.126000673|            458|              0|  152.6666666667| 264.4264232888|1392.8774456641|16.7816559719|68101.5714285714|115977.215079597|      238034|          16|     476711|       119177.75| 137207.36560009|     238274|         78|     238033|        119016.5|168007.864103143|     237816|        217|            0|            0|            0|            0|           168|           104|10.4885349824|6.2931209894|          0|        458| 73.7777777778|159.3783060659|25401.4444444444|           0|           0|           0|           1|           0|           0|             0|           0|            0|          83.0|            41.2|  152.6666666667|             0|             0|               0|             0|             0|               0|               5|             206|               3|             458|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       476616|           5|           3|            211|            463|            211|              0|            42.2|  94.3620686505|            463|              0|  154.3333333333| 267.3131746348|1414.1363277775|16.7850009232|         68088.0|115553.073301117|      237285|           8|     476616|        119154.0|136576.972942001|     237558|          8|     237660|        118830.0|167563.093937776|     237315|        345|            0|            0|            0|            0|           168|           104| 10.490625577|6.2943753462|          0|        463| 74.8888888889|161.4058893322|26051.8611111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|         84.25|            42.2|  154.3333333333|             0|             0|               0|             0|             0|               0|               5|             211|               3|             463|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       477161|           5|           3|            211|            463|            211|              0|            42.2|  94.3620686505|            463|              0|  154.3333333333| 267.3131746348|1412.5211406632|16.7658295628|68165.8571428572|116324.530541969|      238504|          12|     477161|       119290.25|137729.560342905|     238719|         12|     238618|        119309.0|168449.805841384|     238421|        197|            0|            0|            0|            0|           168|           104|10.4786434767| 6.287186086|          0|        463| 74.8888888889|161.4058893322|26051.8611111111|           0|           0|           0|           1|           0|           0|             0|           0|            0|         84.25|            42.2|  154.3333333333|             0|             0|               0|             0|             0|               0|               5|             211|               3|             463|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       474670|           5|           3|            214|            466|            214|              0|            42.8|   95.703709437|            466|              0|  155.3333333333| 269.0452254424|1432.5742094508|16.8538142288|         67810.0|115371.262364883|      236717|          15|     474670|        118667.5|136423.221362787|     236894|         55|     236992|        118496.0| 167297.22178805|     236793|        199|            0|            0|            0|            0|           168|           104| 10.533633893|6.3201803358|          0|        466| 75.5555555556|162.6246530443|26446.7777777778|           0|           0|           0|           1|           0|           0|             0|           0|            0|          85.0|            42.8|  155.3333333333|             0|             0|               0|             0|             0|               0|               5|             214|               3|             466|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       476608|           5|           3|            209|            461|            209|              0|            41.8|  93.4676414595|            461|              0|  153.6666666667| 266.1584740964|1405.7674231234|16.7852826642|68086.8571428572|116165.357657993|      238154|          21|     476608|        119152.0|137536.399945614|     238349|         28|     238442|        119221.0|168305.556058022|     238231|        211|            0|            0|            0|            0|           168|           104|10.4908016651|6.2944809991|          0|        461| 74.4444444444|160.5942955954|25790.5277777778|           0|           0|           0|           1|           0|           0|             0|           0|            0|         83.75|            41.8|  153.6666666667|             0|             0|               0|             0|             0|               0|               5|             209|               3|             461|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       479249|           5|           3|            215|            467|            215|              0|            43.0|  96.1509230325|            467|              0|  155.6666666667| 269.6225757116|1423.0598290242|16.6927839182|68464.1428571429|116522.185511928|      239061|          18|     479249|       119812.25|137797.592674606|     239270|         21|     239238|        119619.0|168903.768394906|     239052|        186|            0|            0|            0|            0|           168|           104|10.4329899489|6.2597939693|          0|        467| 75.7777777778|163.0312683029|26579.1944444444|           0|           0|           0|           1|           0|           0|             0|           0|            0|         85.25|            43.0|  155.6666666667|             0|             0|               0|             0|             0|               0|               5|             215|               3|             467|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "|      80|       6|       475967|           5|           3|            215|            467|            215|              0|            43.0|  96.1509230325|            467|              0|  155.6666666667| 269.6225757116|1432.8724470394|16.8078879418|67995.2857142857|115929.081086548|      237703|          16|     475967|       118991.75|137195.538016305|     237904|         21|     237915|        118957.5|167975.337191208|     237734|        181|            0|            0|            0|            0|           168|           104|10.5049299636|6.3029579782|          0|        467| 75.7777777778|163.0312683029|26579.1944444444|           0|           0|           0|           1|           0|           0|             0|           0|            0|         85.25|            43.0|  155.6666666667|             0|             0|               0|             0|             0|               0|               5|             215|               3|             467|            14480|              219|                1|              32|        0.0|       0.0|         0|         0|         0.0|           0.0|       0|       0|Benign|(78,[0,1,2,3,4,5,...|\n",
            "+--------+--------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+---------------+-------------+----------------+----------------+------------+------------+-----------+----------------+----------------+-----------+-----------+-----------+----------------+----------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------+------------+-----------+-----------+--------------+--------------+----------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+--------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+------------+--------------+--------+--------+------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_idSgu04GGd"
      },
      "source": [
        "from pyspark.ml.feature import StringIndexer\n",
        "from pyspark.ml.feature import MinMaxScaler\n",
        "\n",
        "final_data = training.select(\"independent_features\",\"Label\")\n",
        "\n",
        "scaler = MinMaxScaler(inputCol=\"independent_features\", outputCol=\"scaled_features\")\n",
        "final_data = scaler.fit(final_data).transform(final_data)\n",
        "\n",
        "label_encoder = StringIndexer(inputCol = 'Label',outputCol = 'encoded_label')\n",
        "final_data = label_encoder.fit(final_data).transform(final_data)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8UVWChZsEhLI",
        "outputId": "ef337009-5392-44cc-b509-03c41bb0b7f8"
      },
      "source": [
        "final_data.show()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+------+--------------------+-------------+\n",
            "|independent_features| Label|     scaled_features|encoded_label|\n",
            "+--------------------+------+--------------------+-------------+\n",
            "|(78,[2,3,16,17,18...|Benign|(78,[2,3,15,16,17...|          0.0|\n",
            "|(78,[2,3,16,17,18...|Benign|(78,[2,3,15,16,17...|          0.0|\n",
            "|(78,[2,3,16,17,18...|Benign|(78,[2,3,15,16,17...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[3.35708726900950...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[3.35708726900950...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[3.35708726900950...|          0.0|\n",
            "|(78,[2,3,16,17,18...|Benign|(78,[2,3,15,16,17...|          0.0|\n",
            "|(78,[2,3,16,17,18...|Benign|(78,[2,3,15,16,17...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "|(78,[0,1,2,3,4,5,...|Benign|[0.00122075900691...|          0.0|\n",
            "+--------------------+------+--------------------+-------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLur65_9BMMD"
      },
      "source": [
        "train_data, valid_data = final_data.randomSplit([0.8,0.2])"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w66zOTXMA9wy"
      },
      "source": [
        "#### MultiLayer Perceptron"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aLvHeGqIvHt"
      },
      "source": [
        "layers = [len(feature_assembler.getInputCols()), 4, 2, 3]"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ERegMwmGI400",
        "outputId": "38902600-5c1d-4fcb-c981-ec85992bd9ed"
      },
      "source": [
        "%%time\n",
        "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
        "\n",
        "classifier = MultilayerPerceptronClassifier(labelCol='encoded_label',\n",
        "                                            featuresCol='scaled_features',\n",
        "                                            maxIter=100,\n",
        "                                            layers=layers,\n",
        "                                            blockSize=128,\n",
        "                                            seed=1234)\n",
        "classifier = classifier.fit(train_data)\n",
        "mlp_predictions = classifier.transform(valid_data)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 3.29 s, sys: 391 ms, total: 3.68 s\n",
            "Wall time: 9min 26s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AUhHpv_9FmZZ"
      },
      "source": [
        "#### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVDm6w634QTg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6819d74-474e-47ad-b62b-d498ce872fc4"
      },
      "source": [
        "%%time\n",
        "\n",
        "from pyspark.ml.classification import RandomForestClassifier,DecisionTreeClassifier,MultilayerPerceptronClassifier\n",
        "train_data, valid_data = final_data.randomSplit([0.8,0.2])\n",
        "\n",
        "rf = RandomForestClassifier(featuresCol = 'independent_features', \n",
        "                            labelCol = 'encoded_label' )\n",
        "\n",
        "model = rf.fit(train_data)\n",
        "rf_predictions = model.transform(valid_data)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 899 ms, sys: 106 ms, total: 1.01 s\n",
            "Wall time: 2min 39s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WNI4kz6VF947"
      },
      "source": [
        "#### Decision Tree Algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjBz3ZnpGEAQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67184390-7a2e-45fc-cfc2-a59c30caf144"
      },
      "source": [
        "%%time\n",
        "gbt = DecisionTreeClassifier(labelCol=\"encoded_label\", \n",
        "                    featuresCol=\"independent_features\")\n",
        "gb_ = gbt.fit(train_data)\n",
        "gb_predictions = gb_.transform(valid_data)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 731 ms, sys: 93.7 ms, total: 825 ms\n",
            "Wall time: 2min 11s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2lSg7ZjZPvhg"
      },
      "source": [
        " #### Naive Bayes "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y7XVF8p_8cFz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7622c59d-37e3-4e80-a5b9-06d17106be19"
      },
      "source": [
        "%%time\n",
        "from pyspark.ml.classification import NaiveBayes\n",
        "nb = NaiveBayes(smoothing=1.0, modelType=\"multinomial\",labelCol=\"encoded_label\", \n",
        "                    featuresCol=\"scaled_features\")\n",
        "\n",
        "nb_model = nb.fit(train_data)\n",
        "nb_predictions = nb_model.transform(valid_data)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 305 ms, sys: 34.5 ms, total: 339 ms\n",
            "Wall time: 45 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idqYAX-tN49z"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nt3NqSXYjE-W"
      },
      "source": [
        "NAMES = [\n",
        "        'RF','GBT', 'NB','MLP'\n",
        "]\n",
        "DATA =[\n",
        "      rf_predictions,gb_predictions,nb_predictions,mlp_predictions\n",
        "]\n",
        "\n",
        "METRICS = [\n",
        "           'f1',\n",
        "           'precisionByLabel',\n",
        "           'recallByLabel',\n",
        "           'accuracy'\n",
        "]\n"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ycbq5aCJiwAg"
      },
      "source": [
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "def evaluate(data,metric_name):\n",
        "  evaluator = MulticlassClassificationEvaluator(labelCol = \"encoded_label\",\n",
        "                                              predictionCol = \"prediction\")\n",
        "  evaluator.setMetricName(metric_name)\n",
        "  print(f\"################ {metric_name} #################\")\n",
        "  for i in range(len(data)):\n",
        "    print(f\"{NAMES[i]} : {evaluator.evaluate(data[i])}\")\n"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTV22XOJkoS2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6616ca8c-d74f-4080-c3e2-2068f6bf15cb"
      },
      "source": [
        "for i in METRICS:\n",
        "  evaluate(DATA, i)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "################ f1 #################\n",
            "RF : 0.9999141550458501\n",
            "GBT : 0.9999618494145136\n",
            "NB : 0.8936871068237494\n",
            "MLP : 0.9962710266951234\n",
            "################ precisionByLabel #################\n",
            "RF : 0.9999249699879952\n",
            "GBT : 1.0\n",
            "NB : 0.8751777270773402\n",
            "MLP : 1.0\n",
            "################ recallByLabel #################\n",
            "RF : 0.9999849930968245\n",
            "GBT : 0.9999849930968245\n",
            "NB : 0.9930067831202353\n",
            "MLP : 0.9941716111707035\n",
            "################ accuracy #################\n",
            "RF : 0.9999141610434202\n",
            "GBT : 0.9999618493526312\n",
            "NB : 0.9055151529602518\n",
            "MLP : 0.996263075651214\n"
          ]
        }
      ]
    }
  ]
}