ose_data_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PRE PROCESSING DATA PIPELINE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### IMPORTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/DATASET/envs/p24hasle/miniconda/envs/repro_env/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import xarray as xr\n",
    "import copernicusmarine\n",
    "\n",
    "from src.data_pipeline.ose_data_pipeline import download_copernicus_data_for_sat, filt_daily_ssh_data, grid_input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REAL DATA TEST\n",
    "\n",
    "### Pipeline must:\n",
    "\n",
    "- #### **Download**, **concatenate** and **grid** the *input* nadirs data\n",
    "- #### **Download** and **concatenate** the *reference* nadir data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VALUES TO BE DEFINED:\n",
    "\n",
    "***RUN ONLY THE CELL CONTAINING THE DATA YOU ARE INTERESTED IN***\n",
    "\n",
    "*If you want to include a new dataset to be processed, create a new cell and define all the necessary values*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2023 REPROCESSED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TIME VALUES\n",
    "min_time = '2023-01-01'\n",
    "max_time = '2023-12-31'\n",
    "\n",
    "# LAT LON VALUES\n",
    "min_lon = -66\n",
    "max_lon = -54\n",
    "min_lat = 32\n",
    "max_lat = 44\n",
    "\n",
    "# DATASET ID\n",
    "copernicus_dataset_id = \"cmems_obs-sl_glo_phy-ssh_my_{}-l3-duacs_PT1S\"\n",
    "\n",
    "# SATELLITES NAMES TO DOWNLOAD\n",
    "satellites = []\n",
    "ref_satellites = ['s6a-lr']\n",
    "\n",
    "# DIR TO STORE DOWNLOADED DATA\n",
    "download_sat_input_dir = 'data/dl/2023_reprocessed/input/{}'\n",
    "download_sat_ref_dir = 'data/dl/2023_reprocessed/ref/{}'\n",
    "\n",
    "# DIR TO STORE CONCATENATED DATA\n",
    "concatenated_input_path = \"data/concat/concatenated_input_2023_reprocessed.nc\"\n",
    "concatenated_ref_path = \"data/concat/concatenated_ref_2023_reprocessed.nc\"\n",
    "\n",
    "# GRIDDED INPUT FILEPATH\n",
    "gridded_input_path = \"data/gridded/input_test_6sat_2023_reprocessed.nc\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2023 NRT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TIME VALUES\n",
    "min_time = '2023-01-01'\n",
    "max_time = '2023-12-31'\n",
    "\n",
    "# LAT LON VALUES\n",
    "min_lon = -66\n",
    "max_lon = -54\n",
    "min_lat = 32\n",
    "max_lat = 44\n",
    "\n",
    "# DATASET ID\n",
    "copernicus_dataset_id = \"cmems_obs-sl_glo_phy-ssh_nrt_{}-l3-duacs_PT1S\"\n",
    "\n",
    "# SATELLITES NAMES TO DOWNLOAD\n",
    "satellites = ['al', 'c2n', 'h2b', 'j3n', 's3a', 's3b']\n",
    "ref_satellites= ['s6a-hr', 'swon']\n",
    "\n",
    "# DIR TO STORE DOWNLOADED DATA\n",
    "download_sat_input_dir = 'data/dl/2023/input/{}'\n",
    "download_sat_ref_dir = 'data/dl/2023/ref/{}'\n",
    "\n",
    "# DIR TO STORE CONCATENATED DATA\n",
    "concatenated_input_path = \"data/concat/concatenated_input_2023.nc\"\n",
    "concatenated_ref_path = \"data/concat/concatenated_ref_2023.nc\"\n",
    "\n",
    "# GRIDDED INPUT FILEPATH\n",
    "gridded_input_path = \"data/gridded/input_test_6sat_2023.nc\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2022 NRT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TIME VALUES\n",
    "min_time = '2022-01-01'\n",
    "max_time = '2022-12-31'\n",
    "\n",
    "# LAT LON VALUES\n",
    "min_lon = -66\n",
    "max_lon = -54\n",
    "min_lat = 32\n",
    "max_lat = 44\n",
    "\n",
    "# DATASET ID\n",
    "copernicus_dataset_id = \"cmems_obs-sl_glo_phy-ssh_nrt_{}-l3-duacs_PT1S\"\n",
    "\n",
    "# SATELLITES NAMES TO DOWNLOAD\n",
    "satellites = ['al', 'c2n', 'h2b', 'j3n', 's3a', 's3b']\n",
    "ref_satellites= ['s6a-hr', 'swon']\n",
    "\n",
    "# DIR TO STORE DOWNLOADED DATA\n",
    "download_sat_input_dir = 'data/dl/2022/input/{}'\n",
    "download_sat_ref_dir = 'data/dl/2022/ref/{}'\n",
    "\n",
    "# DIR TO STORE CONCATENATED DATA\n",
    "concatenated_input_path = \"data/concat/concatenated_input_2022.nc\"\n",
    "concatenated_ref_path = \"data/concat/concatenated_ref_2022.nc\"\n",
    "\n",
    "# GRIDDED INPUT FILEPATH\n",
    "gridded_input_path = \"data/gridded/input_test_6sat_2022.nc\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2017 PREPROCESSED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TIME VALUES\n",
    "min_time = '2017-01-01'\n",
    "max_time = '2017-12-31'\n",
    "\n",
    "# LAT LON VALUES\n",
    "min_lon = -66\n",
    "max_lon = -54\n",
    "min_lat = 32\n",
    "max_lat = 44\n",
    "\n",
    "# DATASET ID WITH BRACKETS FOR IN PLACE OF SATELLITE NAME\n",
    "copernicus_dataset_id = \"cmems_obs-sl_glo_phy-ssh_my_{}-l3-duacs_PT1S\"\n",
    "\n",
    "# SATELLITES NAMES TO DOWNLOAD\n",
    "satellites = ['alg', 'h2ag', 'j2g', 'j2n', 'j3', 's3a']\n",
    "ref_satellites = ['c2']\n",
    "\n",
    "# DIR TO STORE DOWNLOADED DATA WITH BRACKETS FOR IN PLACE OF SATELLITE NAME\n",
    "download_sat_input_dir = 'data/dl/2017/input/{}'\n",
    "download_sat_ref_dir = 'data/dl/2017/ref/{}'\n",
    "\n",
    "# DIR TO STORE CONCATENATED DATA\n",
    "concatenated_input_path = \"data/concat/concatenated_input_2017.nc\"\n",
    "concatenated_ref_path = \"data/concat/concatenated_ref_2017.nc\"\n",
    "\n",
    "# GRIDDED INPUT FILEPATH\n",
    "gridded_input_path = \"data/gridded/input_test_6sat_2017.nc\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Login with your copernicusmarine account - can be created [here](https://data.marine.copernicus.eu/register)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "copernicusmarine.login(username=\"<your_username>\", password=\"<your_password>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DOWNLOAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:10<00:00,  3.47s/it]\n",
      "INFO - 2024-07-18T10:59:21Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T10:59:21Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T10:59:21Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T10:59:21Z - Downloading using service original-files...\n",
      "100%|██████████| 919/919 [06:33<00:00,  2.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n",
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:10<00:00,  3.59s/it]\n",
      "INFO - 2024-07-18T11:06:08Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T11:06:08Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T11:06:08Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T11:06:08Z - Downloading using service original-files...\n",
      "100%|██████████| 929/929 [06:36<00:00,  2.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n",
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:11<00:00,  3.94s/it]\n",
      "INFO - 2024-07-18T11:12:59Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T11:12:59Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T11:12:59Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T11:12:59Z - Downloading using service original-files...\n",
      "100%|██████████| 904/904 [06:57<00:00,  2.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n",
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:11<00:00,  3.88s/it]\n",
      "INFO - 2024-07-18T11:20:10Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T11:20:10Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T11:20:10Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T11:20:10Z - Downloading using service original-files...\n",
      "100%|██████████| 808/808 [06:23<00:00,  2.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n",
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:12<00:00,  4.21s/it]\n",
      "INFO - 2024-07-18T11:26:48Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T11:26:49Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T11:26:49Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T11:26:49Z - Downloading using service original-files...\n",
      "100%|██████████| 930/930 [07:08<00:00,  2.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n",
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:11<00:00,  3.82s/it]\n",
      "INFO - 2024-07-18T11:34:11Z - Dataset version was not specified, the latest one was selected: \"202311\"\n",
      "INFO - 2024-07-18T11:34:11Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T11:34:11Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T11:34:11Z - Downloading using service original-files...\n",
      "100%|██████████| 930/930 [07:18<00:00,  2.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n"
     ]
    }
   ],
   "source": [
    "# INPUT DATA\n",
    "for sat in satellites:\n",
    "    download_copernicus_data_for_sat(sat=sat, download_dir=download_sat_input_dir.format(sat), min_time=min_time, max_time=max_time, copernicus_dataset_id=copernicus_dataset_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:10<00:00,  3.45s/it]\n",
      "INFO - 2024-07-18T10:54:49Z - Dataset version was not specified, the latest one was selected: \"202207\"\n",
      "INFO - 2024-07-18T10:54:49Z - Dataset part was not specified, the first one was selected: \"default\"\n",
      "INFO - 2024-07-18T10:54:49Z - Service was not specified, the default one was selected: \"original-files\"\n",
      "INFO - 2024-07-18T10:54:49Z - Downloading using service original-files...\n",
      "100%|██████████| 349/349 [02:56<00:00,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting output validation\n",
      "Succesfully validated output\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# REFERENCE DATA\n",
    "for sat in ref_satellites:\n",
    "    download_copernicus_data_for_sat(sat=sat, download_dir=download_sat_ref_dir.format(sat), min_time=min_time, max_time=max_time, copernicus_dataset_id=copernicus_dataset_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CONCATENATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting\n",
      "Starting input validation\n",
      "Succesfully validated input\n",
      "Starting output validation\n",
      "Succesfully validated output\n"
     ]
    }
   ],
   "source": [
    "# INPUT DATA\n",
    "filt_daily_ssh_data(\n",
    "    input_dir=download_sat_input_dir.format(''),\n",
    "    output_path=concatenated_input_path,\n",
    "    min_time=min_time,\n",
    "    max_time=max_time,\n",
    "    min_lon=min_lon,\n",
    "    max_lon=max_lon,\n",
    "    min_lat=min_lat,\n",
    "    max_lat=max_lat\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting\n",
      "Starting input validation\n",
      "Succesfully validated input\n",
      "Starting output validation\n",
      "Succesfully validated output\n"
     ]
    }
   ],
   "source": [
    "# REFERENCE DATA\n",
    "filt_daily_ssh_data(\n",
    "    input_dir=download_sat_ref_dir.format(''),\n",
    "    output_path=concatenated_ref_path,\n",
    "    min_time=min_time,\n",
    "    max_time=max_time,\n",
    "    min_lon=min_lon,\n",
    "    max_lon=max_lon,\n",
    "    min_lat=min_lat,\n",
    "    max_lat=max_lat\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GRIDDING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# INPUT DATA\n",
    "grid_input(\n",
    "    input_path=concatenated_input_path,\n",
    "    output_path=gridded_input_path,\n",
    "    min_time=min_time,\n",
    "    max_time=max_time,\n",
    "    min_lon=min_lon,\n",
    "    max_lon=max_lon,\n",
    "    min_lat=min_lat,\n",
    "    max_lat=max_lat,\n",
    "    degrees=0.083\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TESTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gridded_input_data = xr.load_dataset(gridded_input_path)\n",
    "concat_ref_data = xr.load_dataset(concatenated_ref_path)\n",
    "print('GRIDDED INPUT\\nMIN TIME: {}\\nMAX TIME: {}'.format(gridded_input_data.time.values.min(), gridded_input_data.time.values.max()))\n",
    "print('\\n\\nCONCAT REF\\nMIN TIME: {}\\nMAX TIME: {}'.format(concat_ref_data.time.values.min(), concat_ref_data.time.values.max()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "4dvarnet-starter",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}