Copy_of_Spark_Titanic_Dataset(Pipelining).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Spark_Titanic_Dataset(Pipelining)",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_XvdLcAuGn9",
        "outputId": "077677ba-379e-4889-fe34-4f8ac2aa8916"
      },
      "source": [
        "pip install pyspark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f0/26/198fc8c0b98580f617cb03cb298c6056587b8f0447e20fa40c5b634ced77/pyspark-3.0.1.tar.gz (204.2MB)\n",
            "\u001b[K     |████████████████████████████████| 204.2MB 73kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/b6/6a4fb90cd235dc8e265a6a2067f2a2c99f0d91787f06aca4bcf7c23f3f80/py4j-0.10.9-py2.py3-none-any.whl (198kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 43.0MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.0.1-py2.py3-none-any.whl size=204612242 sha256=c15c3baa32ab8d7a0fe5a6e8a023137e7f7265df1841f85a56f00f479a043374\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/bd/07/031766ca628adec8435bb40f0bd83bb676ce65ff4007f8e73f\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-YnlPuPwufKl"
      },
      "source": [
        "from pyspark.sql import SparkSession"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "05iCbHNZuuqk"
      },
      "source": [
        "spark=SparkSession.builder.appName('titanic').getOrCreate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4d9g719YvsI7"
      },
      "source": [
        "data=spark.read.csv('/content/sample_data/titanic.csv',inferSchema=True,header=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ve69H-F8wIMn",
        "outputId": "6422b0da-b147-46d7-b628-4ea1a91232f2"
      },
      "source": [
        "data.printSchema()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Survived: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- Age: double (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dANMtC5CwLHr",
        "outputId": "1dbec478-c999-497c-830c-348bdff634b1"
      },
      "source": [
        "data.head(6)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(PassengerId=1, Survived=0, Pclass=3, Name='Braund, Mr. Owen Harris', Sex='male', Age=22.0, SibSp=1, Parch=0, Ticket='A/5 21171', Fare=7.25, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=2, Survived=1, Pclass=1, Name='Cumings, Mrs. John Bradley (Florence Briggs Thayer)', Sex='female', Age=38.0, SibSp=1, Parch=0, Ticket='PC 17599', Fare=71.2833, Cabin='C85', Embarked='C'),\n",
              " Row(PassengerId=3, Survived=1, Pclass=3, Name='Heikkinen, Miss. Laina', Sex='female', Age=26.0, SibSp=0, Parch=0, Ticket='STON/O2. 3101282', Fare=7.925, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=4, Survived=1, Pclass=1, Name='Futrelle, Mrs. Jacques Heath (Lily May Peel)', Sex='female', Age=35.0, SibSp=1, Parch=0, Ticket='113803', Fare=53.1, Cabin='C123', Embarked='S'),\n",
              " Row(PassengerId=5, Survived=0, Pclass=3, Name='Allen, Mr. William Henry', Sex='male', Age=35.0, SibSp=0, Parch=0, Ticket='373450', Fare=8.05, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=6, Survived=0, Pclass=3, Name='Moran, Mr. James', Sex='male', Age=None, SibSp=0, Parch=0, Ticket='330877', Fare=8.4583, Cabin=None, Embarked='Q')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yCE_Gt5gw8oS",
        "outputId": "346cfc19-3c51-4ec8-e6ec-3d938474869c"
      },
      "source": [
        "data.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['PassengerId',\n",
              " 'Survived',\n",
              " 'Pclass',\n",
              " 'Name',\n",
              " 'Sex',\n",
              " 'Age',\n",
              " 'SibSp',\n",
              " 'Parch',\n",
              " 'Ticket',\n",
              " 'Fare',\n",
              " 'Cabin',\n",
              " 'Embarked']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uwaBezrExJjC"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "npFi7D0ewSaz"
      },
      "source": [
        "my_data=data.select(['Survived',\r\n",
        " 'Pclass','Sex',\r\n",
        " 'Age',\r\n",
        " 'SibSp',\r\n",
        " 'Parch','Fare','Embarked'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGR0a4phxWzX"
      },
      "source": [
        "my_cols=my_data.na.drop()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CGA8Me3PxssG"
      },
      "source": [
        "from pyspark.ml.feature import (VectorAssembler,VectorIndexer,\r\n",
        "                                OneHotEncoder,StringIndexer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4CBLw1mx9D_"
      },
      "source": [
        "gender_indexer=StringIndexer(inputCol='Sex',outputCol='SexIndex')\r\n",
        "# A B C\r\n",
        "# 0 1 2\r\n",
        "# one hot encode\r\n",
        "# KEY A B C\r\n",
        "# Example A\r\n",
        "# [1 ,0 ,0]\r\n",
        "gender_encoder=OneHotEncoder(inputCol='SexIndex',outputCol='SexVec')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KcXzZgkH2qcC"
      },
      "source": [
        "embark_indexer=StringIndexer(inputCol='Embarked',outputCol='EmbarkIndex')\r\n",
        "embark_encoder=OneHotEncoder(inputCol='EmbarkIndex',outputCol='EmbarkVec')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3UZdRa1I34bW"
      },
      "source": [
        "assembler=VectorAssembler(inputCols=['Pclass','SexVec','EmbarkVec','Age',\r\n",
        "                                     'SibSp','Parch','Fare'],outputCol='features')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VgUpyVE3kXJ"
      },
      "source": [
        "from pyspark.ml.classification import LogisticRegression"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bZKWscfT4xHJ"
      },
      "source": [
        "from pyspark.ml import Pipeline\r\n",
        "#sets stages for differents steps the data has to cross"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hrF7ddFq4xFx"
      },
      "source": [
        "log_titanic=LogisticRegression(featuresCol='features',labelCol='Survived')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emOFbFtG4xCq"
      },
      "source": [
        "pipeline=Pipeline(stages=[gender_indexer,embark_indexer,\r\n",
        "                          gender_encoder,embark_encoder,\r\n",
        "                          assembler,log_titanic])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGGhh7LuLFq0"
      },
      "source": [
        "train_data,test_data=my_cols.randomSplit([0.7,0.3])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KO44d2FyL5Sj"
      },
      "source": [
        "model=pipeline.fit(train_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2eu8oJfOMGGv"
      },
      "source": [
        "results=model.transform(test_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJ3wqQL6MMQH"
      },
      "source": [
        "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZ4dWcaXMRmQ"
      },
      "source": [
        "my_eval=BinaryClassificationEvaluator(rawPredictionCol='prediction',labelCol='Survived')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bHINl4E_M1m3",
        "outputId": "f96328ee-227e-4ece-9be8-feb4eb63e918"
      },
      "source": [
        "results.select('Survived','prediction').show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------+----------+\n",
            "|Survived|prediction|\n",
            "+--------+----------+\n",
            "|       0|       1.0|\n",
            "|       0|       1.0|\n",
            "|       0|       1.0|\n",
            "|       0|       1.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "|       0|       0.0|\n",
            "+--------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "78cybJnONHTn"
      },
      "source": [
        "AUC=my_eval.evaluate(results)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gv0rgKD1NUcj",
        "outputId": "53085861-eb1b-462d-9f11-b9ad5e9e9563"
      },
      "source": [
        "AUC"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7614863940927117"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fi7KWPLNNXzL"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}