Spark_Samples.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AndrewsJascoro/Andrew_Coronado_0/blob/main/Spark_Samples.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WlWL6cmO-W3y",
        "outputId": "749dfc77-695c-41bb-8977-5c13847bf133"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.5.0.tar.gz (316.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m316.9/316.9 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.5.0-py2.py3-none-any.whl size=317425345 sha256=c1adca7b17384551c3e8125513556482b42d4e0c4678a1d6a86573ccc2c8ffc2\n",
            "  Stored in directory: /root/.cache/pip/wheels/41/4e/10/c2cf2467f71c678cfc8a6b9ac9241e5e44a01940da8fbb17fc\n",
            "Successfully built pyspark\n",
            "Installing collected packages: pyspark\n",
            "Successfully installed pyspark-3.5.0\n"
          ]
        }
      ],
      "source": [
        "pip install pyspark\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# \"From Javier Coronado and AndrewsJascoro GitHub\".\n",
        "# Data Science and Data Analyst.  Studying.\n",
        "# Usando Spark pyspark sql library\n",
        "# Test para for None dato en DataFrame Simple example.\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType\n",
        "from pyspark.sql.functions import col\n",
        "\n",
        "# Create a SparkSession\n",
        "spark = SparkSession.builder.appName(\"example\").getOrCreate()\n",
        "\n",
        "# Create a DataFrame with height, weight, age, and gender\n",
        "data1 = [(1, 144.5, 5.9, 33, 'M'),\n",
        "         (2, 167.2, 5.4, 45, 'M'),\n",
        "         (3, 124.0, 5.1, 22, 'F'),\n",
        "         (4, 118.12, 6.3, 25,'M')]\n",
        "\n",
        "schema1 = StructType([\n",
        "    StructField(\"id\", IntegerType(), True),\n",
        "    StructField(\"height\", FloatType(), True),\n",
        "    StructField(\"weight\", FloatType(), True),\n",
        "    StructField(\"age\", IntegerType(), True),\n",
        "    StructField(\"gender\", StringType(), True)\n",
        "])\n",
        "\n",
        "df1 = spark.createDataFrame(data1, schema=schema1)\n",
        "\n",
        "# Create a DataFrame with id, name, and age\n",
        "data2 = [(1, \"John\", 25),\n",
        "         (2, \"Alice\", None),\n",
        "         (3, \"Bob\", 30), (4, \"Javi\", 25)]\n",
        "\n",
        "schema2 = StructType([\n",
        "    StructField(\"id\", IntegerType(), True),\n",
        "    StructField(\"name\", StringType(), True),\n",
        "    StructField(\"age\", IntegerType(), True)\n",
        "])\n",
        "\n",
        "df2 = spark.createDataFrame(data2, schema=schema2)\n",
        "\n",
        "# Filter rows where the \"age\" column is not null\n",
        "filtered_df = df2.filter(col(\"age\").isNotNull())\n",
        "\n",
        "# Show the resulting DataFrame\n",
        "filtered_df.show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tev1kHNKBtY8",
        "outputId": "90388025-77a2-48e1-ca33-f18b9c0b2d38"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+----+---+\n",
            "| id|name|age|\n",
            "+---+----+---+\n",
            "|  1|John| 25|\n",
            "|  3| Bob| 30|\n",
            "|  4|Javi| 25|\n",
            "+---+----+---+\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNmtfL73Sy93uwWFbNngXp3",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}