From d23c934d0789814e0ead530a11852287c2d8a873 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sun, 24 Mar 2024 16:46:12 +0530 Subject: [PATCH 01/17] added new file of download manager for goes --- satip/goes_download_manager.py | 90 ++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 satip/goes_download_manager.py diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py new file mode 100644 index 00000000..3e47727a --- /dev/null +++ b/satip/goes_download_manager.py @@ -0,0 +1,90 @@ +""" +Script for downloading GOES data. +""" + +import datetime +import logging +import os + +from goes2go import GOES + + +class GOESDownloadManager: + """ + Manager class for downloading GOES data. + """ + def __init__(self, data_dir, log_directory=None): + """ + Initialize the GOESDownloadManager. + + Args: + data_dir (str): Directory to save downloaded GOES data. + log_directory (str, optional): Directory to save logs. + If None, logging is printed to STDOUT. + """ + self.data_dir = data_dir + self.ensure_directory_exists(self.data_dir) + + if log_directory: + self.ensure_directory_exists(log_directory) + logging.basicConfig( + filename=os.path.join(log_directory, 'goes_download.log'), + level=logging.INFO) + else: + logging.basicConfig(level=logging.INFO) + + logging.info(f"GOESDownloadManager initialized. Data will be saved to: {data_dir}") + + @staticmethod + def ensure_directory_exists(directory): + """Ensures the specified directory exists, creating it if necessary.""" + if not os.path.exists(directory): + try: + os.makedirs(directory) + logging.info(f"Created directory: {directory}") + except Exception as e: + logging.error(f"Error creating directory {directory}: {e}") + raise + def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', + domain='F', satellite=16): + """ + Download GOES data for a specified time range and product. + + Args: + start_time (datetime): Start of the download period. + end_time (datetime): End of the download period. + product (str): GOES product identifier. Default is 'ABI-L1b-RadC'. + domain (str): Domain for the product. Default is 'F' (Full Disk). + satellite (int): GOES satellite number. Default is 16. + """ + G = GOES(satellite=satellite, product=product, domain=domain) + current_time = start_time + while current_time <= end_time: + try: + # Download the data + ds = G.nearesttime(current_time) + + # Format the date string for filename + date_string = current_time.strftime("%Y-%m-%d_%H-%M-%S") + filename = f"goes_data_{date_string}.nc" + filepath = os.path.join(self.data_dir, filename) + + # Save to NetCDF + ds.to_netcdf(filepath) + + logging.info(f"Downloaded and saved GOES data to: {filename}") + except Exception as e: + logging.error(f"Error downloading GOES data for {current_time}: {e}") + + current_time += datetime.timedelta(minutes=1) + + logging.info("Completed GOES data download.") + +if __name__ == "__main__": + # Example usage + data_dir = "/path/to/your/data/directory" + log_directory = "/path/to/your/log/directory" + start_time = datetime.datetime(2023, 1, 1, 0, 0) + end_time = datetime.datetime(2023, 1, 1, 1, 0) + manager = GOESDownloadManager(data_dir, log_directory) + manager.download_goes_data(start_time, end_time) From 24a9aa64f3c11eefd86d8696d124658ad7131621 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sun, 24 Mar 2024 16:48:34 +0530 Subject: [PATCH 02/17] added new file of download manager for goes --- satip/goes_download_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py index 3e47727a..cec0c23c 100644 --- a/satip/goes_download_manager.py +++ b/satip/goes_download_manager.py @@ -82,8 +82,8 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', if __name__ == "__main__": # Example usage - data_dir = "/path/to/your/data/directory" - log_directory = "/path/to/your/log/directory" + data_dir = "path to data directory" + log_directory = "path to log directory" start_time = datetime.datetime(2023, 1, 1, 0, 0) end_time = datetime.datetime(2023, 1, 1, 1, 0) manager = GOESDownloadManager(data_dir, log_directory) From ac5d511df3c6b1b0382bcef139fe4cb7fbf6a34a Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sun, 24 Mar 2024 17:30:33 +0530 Subject: [PATCH 03/17] added new file of download manager for goes --- satip/goes_download_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py index cec0c23c..a2388c7f 100644 --- a/satip/goes_download_manager.py +++ b/satip/goes_download_manager.py @@ -81,7 +81,7 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', logging.info("Completed GOES data download.") if __name__ == "__main__": - # Example usage + data_dir = "path to data directory" log_directory = "path to log directory" start_time = datetime.datetime(2023, 1, 1, 0, 0) From 7d515e0c3b13dc9091a8393feed2b773bfe543ae Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sun, 24 Mar 2024 18:55:41 +0530 Subject: [PATCH 04/17] added check for existing file --- satip/goes_download_manager.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py index a2388c7f..8aa45e05 100644 --- a/satip/goes_download_manager.py +++ b/satip/goes_download_manager.py @@ -46,7 +46,7 @@ def ensure_directory_exists(directory): logging.error(f"Error creating directory {directory}: {e}") raise def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', - domain='F', satellite=16): + domain='F', satellite=16): """ Download GOES data for a specified time range and product. @@ -64,11 +64,20 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', # Download the data ds = G.nearesttime(current_time) - # Format the date string for filename - date_string = current_time.strftime("%Y-%m-%d_%H-%M-%S") + # Get acquisition time from the dataset + acquisition_time = ds.time.data.item() + + # Format the acquisition time for filename + date_string = acquisition_time.strftime("%Y-%m-%d_%H-%M-%S") filename = f"goes_data_{date_string}.nc" filepath = os.path.join(self.data_dir, filename) + # Check if data for current acquisition time already exists + if os.path.exists(filepath): + logging.info(f"Data for {date_string} already exists. Skipping.") + current_time += datetime.timedelta(minutes=1) + continue + # Save to NetCDF ds.to_netcdf(filepath) @@ -80,6 +89,7 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', logging.info("Completed GOES data download.") + if __name__ == "__main__": data_dir = "path to data directory" From 30a830c111f54420f9f4fafcdd7f71bc98287af1 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sun, 24 Mar 2024 19:01:12 +0530 Subject: [PATCH 05/17] Determine time increment based on product/domain --- satip/goes_download_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py index 8aa45e05..f6d97929 100644 --- a/satip/goes_download_manager.py +++ b/satip/goes_download_manager.py @@ -59,6 +59,12 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', """ G = GOES(satellite=satellite, product=product, domain=domain) current_time = start_time + + # Determine time increment based on product/domain + time_increment = 1 # Default time increment (minutes) + if product == 'ABI-L1b-RadC' and domain == 'F': + time_increment = 10 + while current_time <= end_time: try: # Download the data @@ -75,7 +81,7 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', # Check if data for current acquisition time already exists if os.path.exists(filepath): logging.info(f"Data for {date_string} already exists. Skipping.") - current_time += datetime.timedelta(minutes=1) + current_time += datetime.timedelta(minutes=time_increment) continue # Save to NetCDF @@ -85,7 +91,7 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', except Exception as e: logging.error(f"Error downloading GOES data for {current_time}: {e}") - current_time += datetime.timedelta(minutes=1) + current_time += datetime.timedelta(minutes=time_increment) logging.info("Completed GOES data download.") From 75a4658feb419d5bb94c89b260d5041c1cbfd639 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 16:13:46 +0530 Subject: [PATCH 06/17] made changes in eumesat.py file --- satip/eumetsat.py | 78 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/satip/eumetsat.py b/satip/eumetsat.py index 2c1ee284..734e4548 100644 --- a/satip/eumetsat.py +++ b/satip/eumetsat.py @@ -17,6 +17,7 @@ import shutil import time import urllib +import warnings import zipfile from io import BytesIO from urllib.error import HTTPError @@ -28,9 +29,18 @@ from satip import utils from satip.data_store import dateset_it_to_filename +from satip.goes_download_manager import GOESDownloadManager log = structlog.stdlib.get_logger() +# Suppress FutureWarning related to 'H' argument +warnings.filterwarnings('ignore', category=FutureWarning) +# constants for different data sources +EUMETSAT_PROVIDER = "EUMETSAT" +GOES_PROVIDER = "GOES" + + + API_ENDPOINT = "https://api.eumetsat.int" # Data Store searching endpoint @@ -196,12 +206,10 @@ def dataset_id_to_link(collection_id, data_id, access_token): ) -class DownloadManager: # noqa: D205 - """ - The DownloadManager class - provides a handler for downloading data from the EUMETSAT API, - managing: retrieval, logging and metadata +class EUMETSATDownloadManager: + """ + Manager class for downloading EUMETSAT data. """ def __init__( @@ -679,3 +687,63 @@ def eumetsat_cloud_name_to_datetime(filename: str): """Takes a file from the EUMETSAT API and returns the it's datetime part for Cloud mask files""" date_str = filename.split("0100-0100-")[-1].split(".")[0] return datetime.datetime.strptime(date_str, "%Y%m%d%H%M%S") + + +class DownloadManager: + """ + Main download manager class to handle both EUMETSAT + + and GOES data downloading based on the provider. + """ + + def __init__(self, provider, user_key=None, + user_secret=None, data_dir=None, + log_directory=None): + """ + Initialize the DownloadManager. + + Args: + provider (str): Provider name ('EUMETSAT' or 'GOES'). + user_key (str): User key for accessing data (for EUMETSAT). + user_secret (str): User secret for accessing data (for EUMETSAT). + data_dir (str): Directory to save downloaded data. + log_directory (str): Directory to save logs. + """ + self.provider = provider + + if self.provider == "EUMETSAT": + self.download_manager = EUMETSATDownloadManager(user_key, user_secret, + data_dir, log_directory) + elif self.provider == "GOES": + self.download_manager = GOESDownloadManager(data_dir, log_directory) + else: + raise ValueError("Invalid provider. Supported providers are 'EUMETSAT' and 'GOES'.") + + def download_data(self, start_time, end_time): + """ + Download data for the specified time range. + + Args: + start_time (datetime): Start of the download period. + end_time (datetime): End of the download period. + """ + if self.provider == "GOES": + self.download_manager.download_goes_data(start_time, end_time) + + +# Example usage +if __name__ == "__main__": + provider = "GOES" + user_key = "your_user_key" + user_secret = "your_user_secret" + data_dir = "path to data directory" + log_directory = "path to log diirectory" + + start_time = datetime.datetime(2024, 3, 1, 0, 0) + end_time = datetime.datetime(2024, 3, 1, 6, 0) + + if data_dir is not None: + manager = DownloadManager(provider, None, None, data_dir, log_directory) + manager.download_data(start_time, end_time) + else: + print("Error: 'data_dir' is not properly set.") From e51177ade4064a8f43ba33b4c27798b1074fb1ec Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 17:57:07 +0530 Subject: [PATCH 07/17] added new file download_manager.py --- satip/download_manager.py | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 satip/download_manager.py diff --git a/satip/download_manager.py b/satip/download_manager.py new file mode 100644 index 00000000..26f2e6be --- /dev/null +++ b/satip/download_manager.py @@ -0,0 +1,90 @@ +"""Satip Download Manager + +This module provides a unified interface for downloading EUMETSAT and GOES +satellite data via the `DownloadManager` class. Users specify the provider +('EUMETSAT' or 'GOES'), and the manager delegates tasks to dedicated +sub-modules for retrieval, storage, and logging. + +Key functionalities: + +* Download data for a specified time range. +* Handle user authentication (for EUMETSAT data). +* Manage data retrieval, storage, and logging for both providers. +""" + +import warnings + +import structlog + +from satip.eumetsat import EUMETSATDownloadManager +from satip.goes_download_manager import GOESDownloadManager + +log = structlog.stdlib.get_logger() + +# Suppress FutureWarning related to 'H' argument +warnings.filterwarnings('ignore', category=FutureWarning) +# constants for different data sources +EUMETSAT_PROVIDER = "EUMETSAT" +GOES_PROVIDER = "GOES" + + + +class DownloadManager: + """ + Main download manager class to handle both EUMETSAT + + and GOES data downloading based on the provider. + + Example usage: + + if __name__ == "__main__": + provider = "GOES" + user_key = "your_user_key" + user_secret = "your_user_secret" + data_dir = "path to data directory" + log_directory = "path to log directory" + + start_time = datetime.datetime(2024, 3, 1, 0, 0) + end_time = datetime.datetime(2024, 3, 1, 6, 0) + + if data_dir is not None: + manager = DownloadManager(provider, None, None, data_dir, log_directory) + manager.download_data(start_time, end_time) + else: + print("Error: 'data_dir' is not properly set.") + + """ + + def __init__(self, provider, user_key=None, + user_secret=None, data_dir=None, + log_directory=None): + """ + Initialize the DownloadManager. + + Args: + provider (str): Provider name ('EUMETSAT' or 'GOES'). + user_key (str): User key for accessing data (for EUMETSAT). + user_secret (str): User secret for accessing data (for EUMETSAT). + data_dir (str): Directory to save downloaded data. + log_directory (str): Directory to save logs. + """ + self.provider = provider + + if self.provider == "EUMETSAT": + self.download_manager = EUMETSATDownloadManager(user_key, user_secret, + data_dir, log_directory) + elif self.provider == "GOES": + self.download_manager = GOESDownloadManager(data_dir, log_directory) + else: + raise ValueError("Invalid provider. Supported providers are 'EUMETSAT' and 'GOES'.") + + def download_data(self, start_time, end_time): + """ + Download data for the specified time range. + + Args: + start_time (datetime): Start of the download period. + end_time (datetime): End of the download period. + """ + if self.provider == "GOES": + self.download_manager.download_goes_data(start_time, end_time) From 87176a1ef379d6de272bcf01c1ba449ee8c6c349 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:01:07 +0530 Subject: [PATCH 08/17] made changes in app.py --- satip/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satip/app.py b/satip/app.py index 88448ffc..5404049c 100644 --- a/satip/app.py +++ b/satip/app.py @@ -14,7 +14,7 @@ import satip from satip import utils -from satip.eumetsat import DownloadManager +from satip.eumetsat import EUMETSATDownloadManager log = structlog.stdlib.get_logger() @@ -137,7 +137,7 @@ def run( ) # 1. Get data from API, download native files with tempfile.TemporaryDirectory() as tmpdir: - download_manager = DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=api_key, user_secret=api_secret, data_dir=tmpdir, From e41a25fd3adec6268697ff0be6f3fed58103f308 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:02:55 +0530 Subject: [PATCH 09/17] minor changes --- satip/goes_download_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/satip/goes_download_manager.py b/satip/goes_download_manager.py index f6d97929..40726601 100644 --- a/satip/goes_download_manager.py +++ b/satip/goes_download_manager.py @@ -94,13 +94,3 @@ def download_goes_data(self, start_time, end_time, product='ABI-L1b-RadC', current_time += datetime.timedelta(minutes=time_increment) logging.info("Completed GOES data download.") - - -if __name__ == "__main__": - - data_dir = "path to data directory" - log_directory = "path to log directory" - start_time = datetime.datetime(2023, 1, 1, 0, 0) - end_time = datetime.datetime(2023, 1, 1, 1, 0) - manager = GOESDownloadManager(data_dir, log_directory) - manager.download_goes_data(start_time, end_time) From 5bdf5b0ccb5ea4138420aee4ff9cbc5dc9d0901f Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:05:23 +0530 Subject: [PATCH 10/17] minor fix in eumetsat.py file --- satip/eumetsat.py | 130 +++++++++++----------------------------------- 1 file changed, 30 insertions(+), 100 deletions(-) diff --git a/satip/eumetsat.py b/satip/eumetsat.py index 734e4548..48d021ba 100644 --- a/satip/eumetsat.py +++ b/satip/eumetsat.py @@ -17,7 +17,6 @@ import shutil import time import urllib -import warnings import zipfile from io import BytesIO from urllib.error import HTTPError @@ -29,17 +28,9 @@ from satip import utils from satip.data_store import dateset_it_to_filename -from satip.goes_download_manager import GOESDownloadManager log = structlog.stdlib.get_logger() -# Suppress FutureWarning related to 'H' argument -warnings.filterwarnings('ignore', category=FutureWarning) -# constants for different data sources -EUMETSAT_PROVIDER = "EUMETSAT" -GOES_PROVIDER = "GOES" - - API_ENDPOINT = "https://api.eumetsat.int" @@ -205,6 +196,36 @@ def dataset_id_to_link(collection_id, data_id, access_token): + access_token ) +def get_filesize_megabytes(filename): + """Returns filesize in megabytes""" + filesize_bytes = os.path.getsize(filename) + return filesize_bytes / 1e6 + + +def eumetsat_filename_to_datetime(inner_tar_name): + """Extracts datetime from EUMETSAT filename. + + Takes a file from the EUMETSAT API and returns + the date and time part of the filename. + + Args: + inner_tar_name: Filename part which contains the datetime information. + + Usage example: + eumetsat_filename_to_datetime(filename) + """ + + p = re.compile(r"^MSG[1234]-SEVI-MSG15-0[01]00-NA-(\d*)\.") + title_match = p.match(inner_tar_name) + date_str = title_match.group(1) + return datetime.datetime.strptime(date_str, "%Y%m%d%H%M%S") + + +def eumetsat_cloud_name_to_datetime(filename: str): + """Takes a file from the EUMETSAT API and returns the it's datetime part for Cloud mask files""" + date_str = filename.split("0100-0100-")[-1].split(".")[0] + return datetime.datetime.strptime(date_str, "%Y%m%d%H%M%S") + class EUMETSATDownloadManager: @@ -656,94 +677,3 @@ def create_and_download_datatailor_data( except Exception as e: log.warn(f"Failed deleting customization {jobID}: {e}", exc_info=True) - - -def get_filesize_megabytes(filename): - """Returns filesize in megabytes""" - filesize_bytes = os.path.getsize(filename) - return filesize_bytes / 1e6 - - -def eumetsat_filename_to_datetime(inner_tar_name): - """Extracts datetime from EUMETSAT filename. - - Takes a file from the EUMETSAT API and returns - the date and time part of the filename. - - Args: - inner_tar_name: Filename part which contains the datetime information. - - Usage example: - eumetsat_filename_to_datetime(filename) - """ - - p = re.compile(r"^MSG[1234]-SEVI-MSG15-0[01]00-NA-(\d*)\.") - title_match = p.match(inner_tar_name) - date_str = title_match.group(1) - return datetime.datetime.strptime(date_str, "%Y%m%d%H%M%S") - - -def eumetsat_cloud_name_to_datetime(filename: str): - """Takes a file from the EUMETSAT API and returns the it's datetime part for Cloud mask files""" - date_str = filename.split("0100-0100-")[-1].split(".")[0] - return datetime.datetime.strptime(date_str, "%Y%m%d%H%M%S") - - -class DownloadManager: - """ - Main download manager class to handle both EUMETSAT - - and GOES data downloading based on the provider. - """ - - def __init__(self, provider, user_key=None, - user_secret=None, data_dir=None, - log_directory=None): - """ - Initialize the DownloadManager. - - Args: - provider (str): Provider name ('EUMETSAT' or 'GOES'). - user_key (str): User key for accessing data (for EUMETSAT). - user_secret (str): User secret for accessing data (for EUMETSAT). - data_dir (str): Directory to save downloaded data. - log_directory (str): Directory to save logs. - """ - self.provider = provider - - if self.provider == "EUMETSAT": - self.download_manager = EUMETSATDownloadManager(user_key, user_secret, - data_dir, log_directory) - elif self.provider == "GOES": - self.download_manager = GOESDownloadManager(data_dir, log_directory) - else: - raise ValueError("Invalid provider. Supported providers are 'EUMETSAT' and 'GOES'.") - - def download_data(self, start_time, end_time): - """ - Download data for the specified time range. - - Args: - start_time (datetime): Start of the download period. - end_time (datetime): End of the download period. - """ - if self.provider == "GOES": - self.download_manager.download_goes_data(start_time, end_time) - - -# Example usage -if __name__ == "__main__": - provider = "GOES" - user_key = "your_user_key" - user_secret = "your_user_secret" - data_dir = "path to data directory" - log_directory = "path to log diirectory" - - start_time = datetime.datetime(2024, 3, 1, 0, 0) - end_time = datetime.datetime(2024, 3, 1, 6, 0) - - if data_dir is not None: - manager = DownloadManager(provider, None, None, data_dir, log_directory) - manager.download_data(start_time, end_time) - else: - print("Error: 'data_dir' is not properly set.") From 74b1de4da34586f7c390f1683628bc90bff73bea Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:32:38 +0530 Subject: [PATCH 11/17] changed name of downloadmanager --- satip/download.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satip/download.py b/satip/download.py index 1c7b67a2..d1057ed8 100644 --- a/satip/download.py +++ b/satip/download.py @@ -28,6 +28,7 @@ import yaml from satip import eumetsat +from satip.eumetsat import EUMETSATDownloadManager from satip.utils import format_dt_str log = structlog.stdlib.get_logger() @@ -110,7 +111,7 @@ def download_eumetsat_data( end_date = datetime.now() # Download the data - dm = eumetsat.DownloadManager(user_key, user_secret, download_directory, download_directory) + dm = EUMETSATDownloadManager(user_key, user_secret, download_directory, download_directory) products_to_use = [] if "rss" in product: products_to_use.append(RSS_ID) @@ -159,7 +160,7 @@ def download_eumetsat_data( def _download_time_range( - x: Tuple[Tuple[datetime, datetime], str, eumetsat.DownloadManager] + x: Tuple[Tuple[datetime, datetime], str, EUMETSATDownloadManager] ) -> None: time_range, product_id, download_manager = x start_time, end_time = time_range From b4e95f7fff86ae505a8098f15a1b88a6f6f5b17a Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:33:15 +0530 Subject: [PATCH 12/17] changed name of downloadmanager --- scripts/extend_gcp_zarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/extend_gcp_zarr.py b/scripts/extend_gcp_zarr.py index 7d7db9a9..fedf950c 100644 --- a/scripts/extend_gcp_zarr.py +++ b/scripts/extend_gcp_zarr.py @@ -2,7 +2,7 @@ import xarray as xr import satpy from satpy import Scene -from satip.eumetsat import DownloadManager +from satip.eumetsat import EUMETSATDownloadManager from satip.scale_to_zero_to_one import ScaleToZeroToOne from satip.serialize import serialize_attrs from satip.utils import convert_scene_to_dataarray @@ -17,7 +17,7 @@ def download_data(last_zarr_time): api_key = os.environ["SAT_API_KEY"] api_secret = os.environ["SAT_API_SECRET"] - download_manager = DownloadManager(user_key=api_key, user_secret=api_secret, data_dir="/mnt/disks/data/native_files/") + download_manager = EUMETSATDownloadManager(user_key=api_key, user_secret=api_secret, data_dir="/mnt/disks/data/native_files/") start_date = pd.Timestamp.utcnow().tz_convert('UTC').to_pydatetime().replace(tzinfo=None) last_zarr_time = pd.Timestamp(last_zarr_time).to_pydatetime().replace(tzinfo=None) start_str = last_zarr_time.strftime("%Y-%m-%d") From 6861eba21b30d6c341c3722934a93cbb36774645 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:33:33 +0530 Subject: [PATCH 13/17] changed name of downloadmanager --- tests/test_eumetsat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_eumetsat.py b/tests/test_eumetsat.py index a210cb79..a6ecfe4d 100644 --- a/tests/test_eumetsat.py +++ b/tests/test_eumetsat.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone, timedelta import pandas as pd -from satip.eumetsat import DownloadManager, eumetsat_filename_to_datetime +from satip.eumetsat import EUMETSATDownloadManager, eumetsat_filename_to_datetime def test_download_manager_setup(): @@ -14,7 +14,7 @@ def test_download_manager_setup(): user_secret = os.environ.get("EUMETSAT_USER_SECRET") with tempfile.TemporaryDirectory() as tmpdirname: - _ = DownloadManager( + _ = EUMETSATDownloadManager( user_key=user_key, user_secret=user_secret, data_dir=tmpdirname, @@ -43,7 +43,7 @@ def test_data_tailor_identify_available_datasets(): end_date = datetime.now(tz=timezone.utc) with tempfile.TemporaryDirectory() as tmpdirname: - download_manager = DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=user_key, user_secret=user_secret, data_dir=tmpdirname, @@ -69,7 +69,7 @@ def test_data_tailor(): end_date = datetime.now(tz=timezone.utc) with tempfile.TemporaryDirectory() as tmpdirname: - download_manager = DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=user_key, user_secret=user_secret, data_dir=tmpdirname, From e54778c12d436c8baee02ca30b9bd3e252f58ce9 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:34:18 +0530 Subject: [PATCH 14/17] changed name of downloadmanager --- scripts/generate_test_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_plots.py b/scripts/generate_test_plots.py index 98211ed6..535a048d 100644 --- a/scripts/generate_test_plots.py +++ b/scripts/generate_test_plots.py @@ -15,7 +15,7 @@ import matplotlib.pyplot as plt import xarray as xr -from satip import eumetsat +from satip import EUMETSATDownloadManager from satip.utils import ( load_cloudmask_to_dataarray, load_native_to_dataarray, @@ -34,7 +34,7 @@ def generate_test_plots(): user_key = os.environ.get("EUMETSAT_USER_KEY") user_secret = os.environ.get("EUMETSAT_USER_SECRET") - download_manager = eumetsat.DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=user_key, user_secret=user_secret, data_dir=os.getcwd(), From 0c9ec7a9a02c1230a863c630b5eb96eaef802f8c Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:34:38 +0530 Subject: [PATCH 15/17] changed name of downloadmanager --- scripts/process_monthly_zarrs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/process_monthly_zarrs.py b/scripts/process_monthly_zarrs.py index 01ae1cfa..0e4d23c8 100644 --- a/scripts/process_monthly_zarrs.py +++ b/scripts/process_monthly_zarrs.py @@ -12,7 +12,7 @@ from satpy import Scene from tqdm import tqdm -from satip.eumetsat import DownloadManager, eumetsat_filename_to_datetime +from satip.eumetsat import EUMETSATDownloadManager, eumetsat_filename_to_datetime from satip.jpeg_xl_float_with_nans import JpegXlFloatWithNaNs from satip.scale_to_zero_to_one import ScaleToZeroToOne from satip.serialize import serialize_attrs @@ -32,7 +32,7 @@ def func(datasets_and_tuples_and_return_data): datasets = [datasets] api_key = os.environ["SAT_API_KEY"] api_secret = os.environ["SAT_API_SECRET"] - download_manager = DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=api_key, user_secret=api_secret, data_dir=tmpdir ) download_manager.download_datasets(datasets) @@ -279,7 +279,7 @@ def create_dummy_zarr(datasets, base_path): date_range = pd.date_range(start="2011-01-01 00:00", end="2019-01-01 00:00", freq="1M") api_key = os.environ["SAT_API_KEY"] api_secret = os.environ["SAT_API_SECRET"] - download_manager = DownloadManager(user_key=api_key, user_secret=api_secret, data_dir="./") + download_manager = EUMETSATDownloadManager(user_key=api_key, user_secret=api_secret, data_dir="./") first = True for date in date_range[::-1]: start_date = pd.Timestamp(date) - pd.Timedelta("1M") From 4121d334cefc82f821566056556f4bb21a10e0ff Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 26 Mar 2024 18:35:47 +0530 Subject: [PATCH 16/17] changed name of downloadmanager --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0eb47ba5..e8aabaad 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -24,6 +24,7 @@ LATEST_DIR_NAME, ) +from satip.eumetsat import EUMETSATDownloadManager USER_KEY = os.environ.get("EUMETSAT_USER_KEY") USER_SECRET = os.environ.get("EUMETSAT_USER_SECRET") RSS_ID = "EO:EUM:DAT:MSG:MSG15-RSS" @@ -38,7 +39,7 @@ def setUp(self) -> None: # noqa D102 if len(list(glob.glob(os.path.join(os.getcwd(), "*.nat")))) == 0: from satip import eumetsat - download_manager = eumetsat.DownloadManager( + download_manager = EUMETSATDownloadManager( user_key=USER_KEY, user_secret=USER_SECRET, data_dir=os.getcwd(), From 323421ddee0562ed9648113d59a4a7e0c79c3850 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 27 Mar 2024 17:44:11 +0530 Subject: [PATCH 17/17] fixed test_utils.py file --- tests/test_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index f6cfd5f0..347f028a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -76,21 +76,23 @@ def test_load_cloudmask_to_dataarray(self): # noqa D102 ) assert isinstance(cloudmask_dataarray, xarray.DataArray) - def test_load_native_to_dataarray(self): # noqa D102 + def test_load_native_to_dataarray(self, setUp): # noqa D102 + rss_filename, _ = setUp for area in ["UK", "RSS"]: rss_dataarray, hrv_dataarray = load_native_to_dataarray( - Path(self.rss_filename), temp_directory=Path(os.getcwd()), area=area + Path(rss_filename), temp_directory=Path(os.getcwd()), area=area ) assert isinstance(rss_dataarray, xarray.DataArray) assert isinstance(hrv_dataarray, xarray.DataArray) - def test_save_dataarray_to_zarr(self): # noqa D102 + def test_save_dataarray_to_zarr(self, setUp): # noqa D102 + rss_filename, _ = setUp # The following is a bit ugly, but since we do not want to lump two tests into one # test function but save_dataarray_to_zarr depends on a dataarray being loaded, # we have to reload the dataarray here. This means that this test can theoretically # fail for two reasons: Either the data-loading failed, or the data-saving failed. rss_dataarray, _ = load_native_to_dataarray( - Path(self.rss_filename), temp_directory=Path(os.getcwd()), area="UK" + Path(rss_filename), temp_directory=Path(os.getcwd()), area="UK" ) zarr_path = os.path.join(os.getcwd(), "tmp.zarr")