PandasBasicsW3schools.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPF3i6WRHzUILDm5UiAi+ai",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mazarrazi/notes_code_log/blob/main/PandasBasicsW3schools.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GMHR_1rQbPBi"
      },
      "outputs": [],
      "source": [
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mydataset = {\n",
        "              'cars' : [\"BWM\", \"Volvo\", \"Ford\"],\n",
        "              'passings' : [3,7,2]\n",
        "            }                 # 2 dimesnsions so dataframe & it is a dictionary\n",
        "\n",
        "myvar = pd.DataFrame(mydataset)\n",
        "\n",
        "print(myvar)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehu2Jz-zbdcj",
        "outputId": "37a7d597-0ea1-4cfc-e52d-3530b38cc75d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    cars  passings\n",
            "0    BWM         3\n",
            "1  Volvo         7\n",
            "2   Ford         2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 1. Pandas Series"
      ],
      "metadata": {
        "id": "KL5gdKhgcEiM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 1.1 Series from a list\n",
        "\n",
        "a = [1, 4, 9]  # 1 dimension so series & it is a list\n",
        "myvar = pd.Series(a)\n",
        "print(myvar)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3BJU23ajfCsB",
        "outputId": "7b311f8b-8fae-4417-ef37-bbb0a888f41d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0    1\n",
            "1    4\n",
            "2    9\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 1.2 we can change the index numbers\n",
        "\n",
        "myvar = pd.Series(a, index = ['w','h','o'])\n",
        "print(myvar)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PfRzAlNafpv-",
        "outputId": "11da7bae-6ec5-404c-d2a0-c15768fb2707"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "w    1\n",
            "h    4\n",
            "o    9\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(myvar['h'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5vDxarIVfu5G",
        "outputId": "7c3da822-a343-4e06-d3b3-1d1394feb4b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 1.3 key/value objects (series from dictionary)\n",
        "\n",
        "afc = {'saka' : 7, 'esr' : 10, 'martinelli' : 11}\n",
        "\n",
        "myvar = pd.Series(afc)\n",
        "\n",
        "print(myvar)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J1MDkqR0gAce",
        "outputId": "de79d339-7e2e-4bc8-c818-bbbd12d618cb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saka           7\n",
            "esr           10\n",
            "martinelli    11\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# NOTE : keys become labels"
      ],
      "metadata": {
        "id": "9BsV-JNDh60I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 1.4 we can select sepcific items by calling their key\n",
        "\n",
        "myvar = pd.Series(afc, index = ['saka', 'esr'])\n",
        "\n",
        "print(myvar)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-DJ8DvWniDqf",
        "outputId": "43aaf3f6-a022-4dd9-ff09-e5c024ec6a8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saka     7\n",
            "esr     10\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# NOTE\n",
        "# while using print()\n",
        "\n",
        "# -for dictionary we use\n",
        "#     dictionary, index = {key}\n",
        "\n",
        "# -for list we use\n",
        "#     list[index]"
      ],
      "metadata": {
        "id": "3TSkfbBxiVIJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "xBI_snMR8hMS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 2. DataFrames\n",
        "#       A Pandas DataFrame is a 2 dimensional data structure,\n",
        "#         like a 2 dimensional array, or a table with rows and columns.\n"
      ],
      "metadata": {
        "id": "bkW0DFBy0Xol"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 2.1 create a Dataframe from two series\n",
        "\n",
        "dataa = {\n",
        "        \"calories\" : [432, 396, 521],\n",
        "        \"duration\" : [50, 40, 45]\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(dataa)   # loading Data into a DataFrame object\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N33pOXok0-fV",
        "outputId": "2e064858-9d23-4bfd-d4dc-9cc96241a6c1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   calories  duration\n",
            "0       432        50\n",
            "1       396        40\n",
            "2       521        45\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2.2.1 Locate Row\n",
        "#       Use loc attribute to return one or more specified rows\n",
        "\n",
        "print(df.loc[0])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5orr7aAD6evn",
        "outputId": "bb4a7d60-cd4a-4b55-de4b-f551d5332e0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "calories    432\n",
            "duration     50\n",
            "Name: 0, dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2.2.2 Locate & return two rows\n",
        "\n",
        "print(df.loc[[0,1]])  # since its two dimensional we use two [[ brackets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lAaYFjcD9F_i",
        "outputId": "e02848dc-ec6d-41e3-fabd-8db9be223a1b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   calories  duration\n",
            "0       432        50\n",
            "1       396        40\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2.3  naming indexes with index argument\n",
        "\n",
        "dataa = {\n",
        "        \"calories\" : [432, 396, 521],\n",
        "        \"duration\" : [50, 40, 45]\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(dataa, index = [\"1st\", \"2nd\", \"3rd\"])\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "Zbz2RjBO9w3h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4e8f2838-059d-43b3-848b-5ef5fc2f5c48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     calories  duration\n",
            "1st       432        50\n",
            "2nd       396        40\n",
            "3rd       521        45\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2.4 locating named index with loc attribute\n",
        "\n",
        "print(df.loc[\"2nd\"])\n",
        "print()\n",
        "print(df.loc[[\"1st\", \"3rd\"]])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eRV-HQ0UlRZG",
        "outputId": "f254699f-1f91-4d65-d2e0-7b85b2f5a10a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "calories    396\n",
            "duration     40\n",
            "Name: 2nd, dtype: int64\n",
            "\n",
            "     calories  duration\n",
            "1st       432        50\n",
            "3rd       521        45\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "ypasn3WimV8Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 3 Read CSV\n",
        "#     CSV files (comma separated files)\n"
      ],
      "metadata": {
        "id": "8jkSE-lvnmPw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 3.1  load CSV into DataFrame\n",
        "\n",
        "# in google collab go to files & then upload to session storage\n",
        "\n",
        "# in jupyter notebook give the local storage file location\n",
        "# Note : while mentioning address use '/' & not '\\')\n",
        "\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "print(df) # If you have a large DataFrame with many rows, Pandas will only return the first 5 rows, and the last 5 rows"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "id": "F1W3C1__n1tO",
        "outputId": "4c327301-1a55-4db4-8e4d-c7714ba76e2e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-bd893ee9ff1c>\u001b[0m in \u001b[0;36m<cell line: 9>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"data.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# If you have a large DataFrame with many rows, Pandas will only return the first 5 rows, and the last 5 rows\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    329\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfind_stack_level\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 )\n\u001b[0;32m--> 331\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;31m# error: \"Callable[[VarArg(Any), KwArg(Any)], Any]\" has no\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 950\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    951\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    603\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m   1440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1443\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m   1733\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1734\u001b[0m                     \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1735\u001b[0;31m             self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1736\u001b[0m                 \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1737\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    854\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    855\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 856\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    857\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    858\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data.csv'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 3.2   to print entire DataFrame use to_string()\n",
        "\n",
        "print(df.to_string())"
      ],
      "metadata": {
        "id": "eUq8nSwspXFn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 3.3  Max Rows\n",
        "#     The number of rows returned is defined in Pandas option settings.\n",
        "\n",
        "print(pd.options.display.max_rows)  # to check system's max allowed rows"
      ],
      "metadata": {
        "id": "BSBDi0iM-9pD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# In my system the number is 60,\n",
        "# which means that if the DataFrame contains more than 60 rows,\n",
        "# the print(df) statement will return only the headers and the first and last 5 rows.\n",
        "\n",
        "# to change the max no. of rows\n",
        "\n",
        "# print(pd.options.display.max_rows = 9999)   # won't work\n",
        "\n",
        "pd.options.display.max_rows = 9999  # changing the max rows\n",
        "\n",
        "df = pd.read_csv('data.csv')\n",
        "\n",
        "print(df)\n"
      ],
      "metadata": {
        "id": "9bevmsnsXxDt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "oY2hMyxxYy89"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 4. Read JSON\n",
        "\n",
        "# JSON is plain text, but has the format of an object\n",
        "\n"
      ],
      "metadata": {
        "id": "BOPVzNOuZIhc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 4.1  Load JSON into DataFrame\n",
        "\n",
        "df = pd.read_json('data2.json')\n",
        "\n",
        "print(df)\n"
      ],
      "metadata": {
        "id": "SgKUatRtaikD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.to_string())  # prints the entire DataFrame"
      ],
      "metadata": {
        "id": "XWO2rSEtgjGa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 4.2  If your JSON code is not in a file, but in a Python Dictionary,\n",
        "#       you can load it into a DataFrame directly\n",
        "\n",
        "# JSON objects have the same format as Python dictionaries.\n",
        "\n",
        "data = {\n",
        "  \"Duration\":{\n",
        "    \"0\":60,\n",
        "    \"1\":60,\n",
        "    \"2\":60,\n",
        "    \"3\":45,\n",
        "    \"4\":45,\n",
        "    \"5\":60\n",
        "  },\n",
        "  \"Pulse\":{\n",
        "    \"0\":110,\n",
        "    \"1\":117,\n",
        "    \"2\":103,\n",
        "    \"3\":109,\n",
        "    \"4\":117,\n",
        "    \"5\":102\n",
        "  },\n",
        "  \"Maxpulse\":{\n",
        "    \"0\":130,\n",
        "    \"1\":145,\n",
        "    \"2\":135,\n",
        "    \"3\":175,\n",
        "    \"4\":148,\n",
        "    \"5\":127\n",
        "  },\n",
        "  \"Calories\":{\n",
        "    \"0\":409,\n",
        "    \"1\":479,\n",
        "    \"2\":340,\n",
        "    \"3\":282,\n",
        "    \"4\":406,\n",
        "    \"5\":300\n",
        "  }\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(data)\n",
        "\n",
        "print(df)\n"
      ],
      "metadata": {
        "id": "gMUFyIW9mG4A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "yMtD8Sflpa1n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 5.  Analyzing DataFrames\n"
      ],
      "metadata": {
        "id": "zgeiYlW0pp6P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 5.1 .head() method returns the headers and a specified number of rows\n",
        "\n",
        "df = pd.read_csv('data.csv')\n",
        "\n",
        "print(df.head())  # by default it prints first 5 rows"
      ],
      "metadata": {
        "id": "HPjcUjc6puaP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# to print first 25 rows\n",
        "\n",
        "print(df.head(25))"
      ],
      "metadata": {
        "id": "fbfdW0Wc7r1G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 5.2  .tail() method to print last 5 rows\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "print(df.tail())"
      ],
      "metadata": {
        "id": "3AVkxoKW8PH4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# to print last 25 rows\n",
        "\n",
        "print(df.tail(25))"
      ],
      "metadata": {
        "id": "xm3QZcxM9e8f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 5.3  .info() to get more information about data\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "print(df.info())"
      ],
      "metadata": {
        "id": "0d5kFkq99rMv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "su79zrle-C-A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6 Data Cleaning\n",
        "\n",
        "#   Data cleaning means fixing bad data in data set.\n",
        "\n",
        "#     Bad data could be:\n",
        "# 6.1 Empty cells\n",
        "# 6.2 Data in wrong format\n",
        "# 6.3 Wrong data\n",
        "# 6.4 Duplicates\n"
      ],
      "metadata": {
        "id": "Y-PweMid_F3o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "80rqrVaOOv0K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.1  Cleaning Empty cells\n",
        "\n",
        "# Empty cells can potentially give wrong result when analyzing data"
      ],
      "metadata": {
        "id": "RYWlzM9RAPjY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.1.1  Remove Rows\n",
        "\n",
        "#   One way to deal with empty cells is to remove rows that contain empty cells.\n",
        "\n",
        "#   This is usually not a problem, since data sets can be very big,\n",
        "#   and removing a few rows will not have a big impact on the result."
      ],
      "metadata": {
        "id": "poLipKF9A3wi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  using .dropna() method\n",
        "\n",
        "#  returns a new DataFrame, and will not change the original.\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "new_df = df.dropna()\n",
        "\n",
        "print(df)\n",
        "print()\n",
        "print(new_df)"
      ],
      "metadata": {
        "id": "xvGRkRc35nQc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  using .dropna(inplace = True)\n",
        "\n",
        "# will NOT return a new DataFrame,\n",
        "# but it will remove all rows containing NULL values from the original DataFrame\n",
        "\n",
        "df = pd.read_csv(\"datatrail.csv\") # note Original data will be altered\n",
        "\n",
        "new_df = df.dropna(inplace = True)\n",
        "\n",
        "print(df)\n",
        "print()\n",
        "print(new_df) # won't be printed"
      ],
      "metadata": {
        "id": "2JiCBWc86SHD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  .dropna() -- returns a new DataFrame, and will not change the original.\n",
        "#               & after a row is removed index not updated in new DataFrame\n",
        "\n",
        "#  .dropna(inplace = True) -- changes the original DataFrame too\n",
        "#                            & will not return a new DataFrame\n"
      ],
      "metadata": {
        "id": "GDdM-AYs7Dyc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df) # look original is also altered (no row 17)"
      ],
      "metadata": {
        "id": "wr5QN9jk7Oc0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  6.1.2   Replace empty values\n",
        "\n",
        "#  next way is to fill the empty cells instead of deleting entire row\n",
        "#  fill the empty cells using .fillna()\n",
        "#  it replaces empty cells with a value\n",
        "\n",
        "df = pd.read_csv(\"datatrail1.csv\")  # will alter original data\n",
        "\n",
        "# new_df = df.fillna() # this won't work\n",
        "\n",
        "df.fillna(130, inplace = True) # replaces all empty cells with 130\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "2ieMMIua7xq8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  Replace only for specified columns\n",
        "\n",
        "df = pd.read_csv(\"datatrail2.csv\") # will alter original data\n",
        "\n",
        "# Replaces NULL values in the \"Calories\" columns with the number 130:\n",
        "df[\"Calories\"].fillna(130, inplace = True)\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "o-vaxo6R-fpI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.1.3  Replace Using Mean, Median, or Mode\n",
        "\n",
        "#  A common way to replace empty cells, is to calculate the mean, median\n",
        "#   or mode value of the column.\n",
        "\n",
        "#  Pandas uses the mean() median() and mode() methods to calculate\n",
        "#   the respective values for a specified column\n",
        "\n"
      ],
      "metadata": {
        "id": "QNO_eX1j_QzE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  Mean\n",
        "\n",
        "df = pd.read_csv(\"data4.csv\") # will alter original data\n",
        "\n",
        "abc = df[\"Calories\"].mean()  # calculatinf mean of only one column\n",
        "\n",
        "df[\"Calories\"].fillna(abc, inplace = True)\n",
        "\n",
        "# to display only two digits after the decimal point\n",
        "df[\"Calories\"] = df[\"Calories\"].round(2)\n",
        "# or you can also use the below line\n",
        "# abc = df[\"Calories\"].mean().round(2)\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "ruznIyryE2sF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Median\n",
        "\n",
        "df = pd.read_csv(\"data5.csv\")\n",
        "\n",
        "abc = df[\"Calories\"].median().round(2)\n",
        "\n",
        "df[\"Calories\"].fillna(abc, inplace = True)\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "e8FhuXI0GCj-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  Mode\n",
        "\n",
        "df = pd.read_csv(\"data6.csv\")\n",
        "\n",
        "abc = df[\"Calories\"].mode()[0].round()  # why [0]?\n",
        "\n",
        "df[\"Calories\"].fillna(abc, inplace = True)\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "IPCpGPA0y4pS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# clarification from the above Mode\n",
        "# why [o]? in\n",
        "# abc = df[\"Calories\"].mode()[0].round()\n",
        "\n",
        "# Reasons:\n",
        "# 1. in mode if there are multiple values with highest frequency\n",
        "#     by using the [0] we can extract only the first mode value\n",
        "\n",
        "# 2. & if the column contains NaN values the .mode() will return NaN\n",
        "#     try the same program without the [0] you will have NaN\n",
        "#      to avoid those errors [0] is used after .mode()[0]\n"
      ],
      "metadata": {
        "id": "HJHkVzxrz7c2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "2oMEYGRf15tB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.2 Data in wrong format\n",
        "\n",
        "# Cells with data of wrong format can make it difficult,\n",
        "#   or even impossible, to analyze data.\n",
        "\n",
        "# To fix it, you have two options:\n",
        "#   remove the rows or\n",
        "#   convert all cells in the columns into the same format.\n"
      ],
      "metadata": {
        "id": "02o87MmNAOWm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  6.2.1  Converting all cells in the columns into the same format\n",
        "\n",
        "# the example needs date & its not in out csv data\n",
        "# w3 schools gave the table instead of the csv\n",
        "\n",
        "# this might come in handy out of the current learning scope\n",
        "\n",
        "# so we have 2 options\n",
        "# both involves copying data from their site first & giving it to chatgpt\n",
        "\n",
        "# option 1  directly using it as nested dictionary\n",
        "# copy & give it to chatgpt, ask it to be give as DataFrame\n",
        "\n",
        "# options 2  copy data & convert it to .csv\n",
        "# copy & give it to chatgpt, ask for csv\n",
        "# it'll give a program\n"
      ],
      "metadata": {
        "id": "iu0-9SI3An0L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Create the DataFrame\n",
        "data = {\n",
        "    'Duration': [60, 60, 60, 45, 45, 60, 60, 450, 30, 60, 60, 60, 60, 60, 60, 60, 60, 60, 45, 60, 45, 60, 45, 60, 45, 60, 60, 60, 60, 60, 60, 60],\n",
        "    'Date': ['2020/12/01', '2020/12/02', '2020/12/03', '2020/12/04', '2020/12/05',\n",
        "             '2020/12/06', '2020/12/07', '2020/12/08', '2020/12/09', '2020/12/10',\n",
        "             '2020/12/11', '2020/12/12', '2020/12/12', '2020/12/13', '2020/12/14',\n",
        "             '2020/12/15', '2020/12/16', '2020/12/17', '2020/12/18', '2020/12/19',\n",
        "             '2020/12/20', '2020/12/21', None, '2020/12/23', '2020/12/24', '2020/12/25',\n",
        "             '20201226','2020/12/27', '2020/12/28', '2020/12/29', '2020/12/30', '2020/12/31'],\n",
        "    'Pulse': [110, 117, 103, 109, 117, 102, 110, 104, 109, 98, 103, 100, 100, 106, 104, 98, 98, 100, 90, 103, 97, 108, 100, 130, 105, 102, 100, 92, 103, 100, 102, 92],\n",
        "    'Maxpulse': [130, 145, 135, 175, 148, 127, 136, 134, 133, 124, 147, 120, 120, 128, 132, 123, 120, 120, 112, 123, 125, 131, 119, 101, 132, 126, 120, 118, 132, 132, 129, 115],\n",
        "    'Calories': [409.1, 479.0, 340.0, 282.4, 406.0, 300.0, 374.0, 253.3, 195.1, 269.0, 329.3, 250.7, 250.7, 345.3, 379.3, 275.0, 215.2, 300.0, None, 323.0, 243.0, 364.2, 282.0, 300.0, 246.0, 334.5, 250.0, 241.0, None, 280.0, 380.3, 243.0]\n",
        "    }\n",
        "\n",
        "\n",
        "df = pd.DataFrame(data)\n",
        "\n",
        "# Save the DataFrame as a CSV file\n",
        "df.to_csv('output.csv', index=False)  # can we give save location in jupyter?\n",
        "\n",
        "print(\"DataFrame saved as 'output.csv'\")\n",
        "\n"
      ],
      "metadata": {
        "id": "zzs_gDlsDniN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df = pd.read_csv(\"outputreuse.csv\")\n",
        "\n",
        "print(df)\n"
      ],
      "metadata": {
        "id": "0BiDrlxjDo5U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  Contd.\n",
        "#  6.2.1  Converting all cells in the columns into the same format\n",
        "\n",
        "# In our Data Frame, we have two cells with the wrong format.\n",
        "#  row 22 and 26, the 'Date' column should be a string that represents a date:\n",
        "\n",
        "# Let's try to convert all cells in the 'Date' column into dates.\n",
        "# Pandas has a to_datetime() method for this\n",
        "\n",
        "\n",
        "df = pd.read_csv(\"outputreuse.csv\")\n",
        "\n",
        "df[\"Date\"] = pd.to_datetime(df[\"Date\"])\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "T1qaKigCLLMF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# the result, the date in row 26 was fixed,\n",
        "#  but the empty date in row 22 got a NaT (Not a Time) value,\n",
        "#  in other words an empty value.\n",
        "#  One way to deal with empty values is simply removing the entire row.\n",
        "\n",
        "df.dropna(subset=[\"Date\"], inplace = True) # method 1\n",
        "# or\n",
        "# df = df[df['Date'].notna()]  # method 2\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "cE84_CNELu-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "abIIHVbpRr08"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.3 Data is Wrong,\n",
        "#  not wrong format or empty cell, data is just wrong\n",
        "\n",
        "# \"Wrong data\" does not have to be \"empty cells\" or \"wrong format\",\n",
        "#  it can just be wrong, like if someone registered \"199\" instead of \"1.99\".\n",
        "\n",
        "# Sometimes you can spot wrong data by looking at the data set,\n",
        "#  because you have an expectation of what it should be.\n",
        "\n",
        "# If you take a look at our data set, you can see that in row 7, the duration is 450,\n",
        "#  but for all the other rows the duration is between 30 and 60.\n",
        "\n",
        "# It doesn't have to be wrong, but taking in consideration that\n",
        "#  this is the data set of someone's workout sessions,\n",
        "#  we conclude with the fact that this person did not work out in 450 minutes.\n",
        "\n",
        "# there are few options\n",
        "#   1.Replace (for small datasets we can replace one by one)\n",
        "#   2.Set boudary, if values crosses boundary replace (loop, for large dataset)\n",
        "#   3.Remove rows"
      ],
      "metadata": {
        "id": "zzVP09iYTmj8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.3.1  Replace one by one\n",
        "\n",
        "# in row 7 it is most likely a typo,\n",
        "#  and the value should be \"45\" instead of \"450\", and we could just insert \"45\"\n",
        "\n",
        "df = pd.read_csv(\"outputreuse1.csv\")\n",
        "\n",
        "df.loc[7, \"Duration\"] = 45\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "Ofy_tIZGYepy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 6.3.2  Set boudary, if values crosses boundary replace (loop, for large dataset)\n",
        "\n",
        "# For small data sets you might be able to replace the wrong data one by one,\n",
        "# but not for big data sets.\n",
        "\n",
        "# To replace wrong data for larger data sets you can create some rules,\n",
        "# e.g. set some boundaries for legal values, and replace any values that are outside of the boundaries.\n",
        "\n",
        "# Loop through all values in the \"Duration\" column.\n",
        "# If the value is higher than 120, set it to 120:\n",
        "\n",
        "df = pd.read_csv(\"outputreuse2.csv\")\n",
        "\n",
        "print(df)\n",
        "print()\n",
        "\n",
        "for numm in df.index:\n",
        "  if df.loc[numm, \"Duration\"] > 120: # [row_index, \"ColumnName\"] is unique to pandas and are not part of core Python\n",
        "    df.loc[numm, \"Duration\"] = 120\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "k06SWVzYZQ9Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  6.3.3  Removing rows\n",
        "\n",
        "#  Another way of handling wrong data is to remove the rows that contains wrong data.\n",
        "\n",
        "#  Deleting rows where \"Duration\" is higher than 120:\n",
        "\n",
        "df = pd.read_csv(\"outputreuse3.csv\")\n",
        "\n",
        "print(df)\n",
        "print()\n",
        "\n",
        "for numm in df.index:\n",
        "  if df.loc[numm, \"Duration\"] > 120:\n",
        "    df.drop(numm, inplace = True)\n",
        "\n",
        "print(df)"
      ],
      "metadata": {
        "id": "oN0zcH2DUD23"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "DwmvvmxbV2sW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 7 Removing Duplicates\n"
      ],
      "metadata": {
        "id": "ylw8NPsV8K-u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  7.1 Discovering Duplicates\n",
        "\n",
        "# To discover duplicates, we can use the duplicated() method.\n",
        "# The duplicated() method returns a Boolean values for each row\n",
        "\n",
        "df = pd.read_csv(\"outputreuse4.csv\")\n",
        "\n",
        "print(df)  # in our data row 11 & 12 are duplicates\n",
        "print()\n",
        "\n",
        "print(df.duplicated())"
      ],
      "metadata": {
        "id": "bZL0nIoQ8VNO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  7.2 Removing Duplicates\n",
        "\n",
        "df.drop_duplicates(inplace = True)\n",
        "\n",
        "print(df)\n",
        "print()\n",
        "\n",
        "print(df.duplicated())"
      ],
      "metadata": {
        "id": "vK3mGBZT8YJe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "1kowf_2Y8--2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 8 Data Correlations\n",
        "\n",
        "# corr() method calculates the relationship between each column in your data set\n",
        "# corr() method ignores non numeric columns\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "df.corr()\n"
      ],
      "metadata": {
        "id": "O2jD_-R1--Ae"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Result explanation from w3schools\n",
        "\n",
        "# The Result of the corr() method is a table with a lot of numbers that represents how well the relationship is between two columns.\n",
        "\n",
        "# The number varies from -1 to 1.\n",
        "# 1 means that there is a 1 to 1 relationship (a perfect correlation), and for this data set, each time a value went up in the first column, the other one went up as well.\n",
        "\n",
        "# 0.9 is also a good relationship, and if you increase one value, the other will probably increase as well.\n",
        "\n",
        "# -0.9 would be just as good relationship as 0.9, but if you increase one value, the other will probably go down.\n",
        "\n",
        "# 0.2 means NOT a good relationship, meaning that if one value goes up does not mean that the other will.\n",
        "\n",
        "\n",
        "# What is a good correlation?\n",
        "# It depends on the use, but I think it is safe to say you have to have at least 0.6 (or -0.6) to call it a good correlation.\n",
        "\n",
        "\n",
        "# Perfect Correlation:\n",
        "# We can see that \"Duration\" and \"Duration\" got the number 1.000000, which makes sense, each column always has a perfect relationship with itself.\n",
        "\n",
        "\n",
        "# Good Correlation:\n",
        "# \"Duration\" and \"Calories\" got a 0.922721 correlation, which is a very good correlation, and we can predict that the longer you work out, the more calories you burn, and the other way around: if you burned a lot of calories, you probably had a long work out.\n",
        "\n",
        "\n",
        "# Bad Correlation:\n",
        "# \"Duration\" and \"Maxpulse\" got a 0.009403 correlation, which is a very bad correlation, meaning that we can not predict the max pulse by just looking at the duration of the work out, and vice versa.\n"
      ],
      "metadata": {
        "id": "6G-gtSYE_BhO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  Result explained by Chatgpt,\n",
        "\n",
        "# table appears to be a correlation matrix for a DataFrame with four columns: Duration, Pulse, Maxpulse, and Calories. In this context, a correlation matrix is used to quantify the linear relationships between pairs of variables. Each cell in the matrix represents the correlation coefficient between two variables.\n",
        "\n",
        "# Here's an explanation of the correlation results based on the values in the matrix:\n",
        "\n",
        "# 1. Duration vs. Duration (1.000000): # The diagonal elements always have a correlation of 1 because they represent the correlation of a variable with itself, which is a perfect correlation. So, the Duration variable is perfectly correlated with itself, as expected.\n",
        "\n",
        "# 2. Duration vs. Pulse (-0.155408): This value is negative, indicating a weak negative correlation between Duration and Pulse. In other words, as the Duration of an activity increases, the Pulse tends to decrease slightly. However, the correlation is weak, so this relationship is not very strong.\n",
        "\n",
        "# 3. Duration vs. Maxpulse (0.009403): This value is close to zero, indicating a very weak correlation between Duration and Maxpulse. There is almost no linear relationship between these two variables.\n",
        "\n",
        "# 4. Duration vs. Calories (0.922717): This value is close to 1, indicating a very strong positive correlation between Duration and Calories. As the Duration of an activity increases, the Calories burned also increase significantly. This strong positive correlation suggests that these two variables move together in a nearly linear fashion.\n",
        "\n",
        "# 5. Pulse vs. Pulse (1.000000): Again, this is a perfect correlation of 1 because it represents the correlation of Pulse with itself.\n",
        "\n",
        "# 6. Pulse vs. Maxpulse (0.786535): This value is positive and relatively high, indicating a strong positive correlation between Pulse and Maxpulse. It suggests that as Pulse increases, Maxpulse also tends to increase.\n",
        "\n",
        "# 7. Pulse vs. Calories (0.025121): This value is close to zero, indicating a very weak correlation between Pulse and Calories. There is almost no linear relationship between these two variables.\n",
        "\n",
        "# 8. Maxpulse vs. Maxpulse (1.000000): Perfect correlation of 1, representing the correlation of Maxpulse with itself.\n",
        "\n",
        "# 9. Maxpulse vs. Calories (0.203813): This value is positive but relatively moderate, indicating a moderate positive correlation between Maxpulse and Calories. It suggests that as Maxpulse increases, Calories burned tend to increase moderately.\n",
        "\n",
        "# 10. Calories vs. Calories (1.000000): Perfect correlation of 1, representing the correlation of Calories with itself.\n",
        "\n",
        "# In summary, the correlation matrix provides insights into how these variables are related. Strong positive correlations (close to 1) suggest that variables move together in the same direction, strong negative correlations (close to -1) suggest they move in opposite directions, and values close to 0 suggest little to no linear relationship."
      ],
      "metadata": {
        "id": "RZPCHV99hix9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# General understanding of all Correlation results,\n",
        "\n",
        "# Understanding correlation results is essential for analyzing relationships between variables in your data. Here are some important points to consider when interpreting correlation results:\n",
        "\n",
        "# 1. **Range of Values**:\n",
        "#    - Correlation coefficients range from -1 to 1.\n",
        "#    - **-1** indicates a perfect negative linear relationship (as one variable increases, the other decreases linearly).\n",
        "#    - **1** indicates a perfect positive linear relationship (as one variable increases, the other increases linearly).\n",
        "#    - **0** indicates no linear relationship; the variables are not linearly related.\n",
        "\n",
        "# 2. **Sign of the Coefficient**:\n",
        "#    - A positive correlation coefficient (between 0 and 1) indicates that the variables tend to increase together.\n",
        "#    - A negative correlation coefficient (between -1 and 0) indicates that as one variable increases, the other tends to decrease.\n",
        "\n",
        "# 3. **Strength of the Correlation**:\n",
        "#    - The magnitude of the correlation coefficient indicates the strength of the relationship.\n",
        "#    - Values close to -1 or 1 indicate a strong linear relationship.\n",
        "#    - Values close to 0 indicate a weak or no linear relationship.\n",
        "\n",
        "# 4. **Perfect Correlation**:\n",
        "#    - A correlation coefficient of 1 or -1 represents a perfect linear relationship. This rarely occurs in real-world data.\n",
        "#    - If the coefficient is exactly 1, it means the variables have a positive linear relationship with a slope of 1.\n",
        "#    - If the coefficient is exactly -1, it means the variables have a negative linear relationship with a slope of -1.\n",
        "\n",
        "# 5. **No Correlation vs. Independence**:\n",
        "#    - A correlation coefficient of 0 indicates no linear relationship between variables.\n",
        "#    - However, it's important to note that a correlation of 0 does not necessarily mean variables are independent; they could have a nonlinear relationship or be related in a different way.\n",
        "\n",
        "# 6. **Scatterplots**:\n",
        "#    - When interpreting correlation results, it's helpful to visualize the data with scatterplots.\n",
        "#    - Scatterplots can reveal nonlinear relationships or patterns that may not be captured by correlation coefficients alone.\n",
        "\n",
        "# 7. **Causation vs. Correlation**:\n",
        "#    - Correlation does not imply causation. Even if two variables have a strong correlation, it doesn't mean that one causes the other.\n",
        "#    - Correlation only quantifies the strength and direction of a linear relationship, not a causal relationship.\n",
        "\n",
        "# 8. **Outliers**:\n",
        "#    - Outliers in your data can significantly affect correlation coefficients.\n",
        "#    - It's important to check for outliers and assess their impact on the relationship.\n",
        "\n",
        "# 9. **Multiple Variables**:\n",
        "#    - Correlation can be computed between pairs of variables, but it doesn't capture complex relationships involving multiple variables.\n",
        "#    - In some cases, correlations can be misleading when considering more than two variables.\n",
        "\n",
        "# 10. **Domain Knowledge**:\n",
        "#     - Always consider domain knowledge and the context of your data when interpreting correlation results. Understanding the data and its meaning is crucial.\n",
        "\n",
        "# In practice, correlation analysis is a useful tool for identifying potential relationships between variables, but it should be supplemented with other analyses and domain expertise to draw meaningful conclusions about your data.\n",
        "\n"
      ],
      "metadata": {
        "id": "vtYCKSY_hlh0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "YSQZ8IgpienV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  9 Plotting\n",
        "\n",
        "# Pandas uses the plot() method to create diagrams.\n",
        "# We can use Pyplot, a submodule of the Matplotlib library to visualize the diagram\n"
      ],
      "metadata": {
        "id": "9XhqzY4rjOvU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 9.1  plot()\n",
        "\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "df = pd.read_csv(\"data.csv\")\n",
        "\n",
        "df.plot()\n",
        "\n",
        "plt.show() # not necessary"
      ],
      "metadata": {
        "id": "YdxV7F9kjR88"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.plot()"
      ],
      "metadata": {
        "id": "x8xbHkgBjtK8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  9.2  Scatter Plot\n",
        "\n",
        "# Specify that you want a scatter plot with the kind argument:\n",
        "# kind = 'scatter'\n",
        "\n",
        "# A scatter plot needs an x-axis and a y-axis.\n",
        "# In the example below we will use \"Duration\" for the x-axis and \"Calories\" for the y-axis.\n",
        "\n",
        "df.plot(kind = 'scatter', x = 'Duration', y = 'Calories')\n"
      ],
      "metadata": {
        "id": "F5SfFVhfj_bb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# we learned that the correlation between \"Duration\" and \"Calories\" was 0.922721,\n",
        "#  and we concluded with the fact that higher duration means more calories burned.\n",
        "#   here we can confirm it\n",
        "\n",
        "\n",
        "# Let's create another scatterplot,\n",
        "#  where there is a bad relationship between the columns,\n",
        "#   like \"Duration\" and \"Maxpulse\", with the correlation 0.009403\n",
        "\n",
        "\n",
        "df.plot(kind = 'scatter', x = 'Duration', y = 'Maxpulse')\n"
      ],
      "metadata": {
        "id": "JerHaPZkkabL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  9.3  Histogram\n",
        "\n",
        "# Use the kind argument to specify that you want a histogram:\n",
        "# kind = 'hist'\n",
        "\n",
        "# A histogram needs only one column.\n",
        "# A histogram shows us the frequency of each interval, e.g. how many workouts lasted between 50 and 60 minutes?\n",
        "# In the example below we will use the \"Duration\" column to create the histogram:\n",
        "\n",
        "df = pd.read_csv('data.csv')\n",
        "\n",
        "df['Duration'].plot(kind = 'hist')\n"
      ],
      "metadata": {
        "id": "GG79wKOSlMT-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  The histogram tells us that there were over 100 workouts that lasted between 50 and 60 minutes."
      ],
      "metadata": {
        "id": "hcbxLDkMlwnD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "CVCFFmdklzW6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}