diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..1da089cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: check-added-large-files diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 154641e4..f5037eaf 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -2,7 +2,7 @@ FROM oraclelinux:8 LABEL author="OPERA ADT" \ description="RTC cal/val release R4" \ - version="1.0.1-final" + version="1.0.2-final" RUN yum -y update &&\ yum -y install curl &&\ diff --git a/Docker/environment.yml b/Docker/environment.yml index 7ee611da..0401f2a4 100644 --- a/Docker/environment.yml +++ b/Docker/environment.yml @@ -10,6 +10,6 @@ dependencies: - pyre>=1.11.2 - scipy!=1.10.0 - - isce3>=0.15.0 + - isce3==0.15.0 # Workaround for the issue with `libabseil` (09/11/2023) - libabseil=20230125.3 \ No newline at end of file diff --git a/Docker/requirements.txt b/Docker/requirements.txt index c5b1b139..2bc9ee0f 100644 --- a/Docker/requirements.txt +++ b/Docker/requirements.txt @@ -21,4 +21,4 @@ setuptools shapely yamale backoff -isce3>=0.14.0 \ No newline at end of file +isce3==0.15.0 \ No newline at end of file diff --git a/build_docker_image.sh b/build_docker_image.sh index 1727937c..4f957468 100755 --- a/build_docker_image.sh +++ b/build_docker_image.sh @@ -2,7 +2,7 @@ REPO=opera IMAGE=rtc -TAG=final_1.0.1 +TAG=final_1.0.2 echo "IMAGE is $REPO/$IMAGE:$TAG" diff --git a/src/rtc/core.py b/src/rtc/core.py index d26cb4b7..bd511df7 100644 --- a/src/rtc/core.py +++ b/src/rtc/core.py @@ -83,7 +83,15 @@ def save_as_cog(filename, scratch_dir='.', logger=None, logger = logging.getLogger('rtc_s1') logger.info(' COG step 1: add overviews') - gdal_ds = gdal.Open(filename, gdal.GA_Update) + + # open GeoTIFF file in update mode + try: + gdal_ds = gdal.Open(filename, gdal.GA_Update) + except RuntimeError: + # fix for GDAL >= 3.8 + gdal_ds = gdal.OpenEx(filename, gdal.GA_Update, + open_options=["IGNORE_COG_LAYOUT_BREAK=YES"]) + gdal_dtype = gdal_ds.GetRasterBand(1).DataType dtype_name = gdal.GetDataTypeName(gdal_dtype).lower() diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 18da198b..81282e95 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -415,6 +415,13 @@ def get_metadata_dict(product_id: str, # If the DEM description is not provided, use DEM source dem_file_description = os.path.basename(cfg_in.dem) + # reformat burst ID to URL data access format used by ASF + # (e.g., from "t018_038602_iw2" to "T018-038602-iw2") + burst_id_asf = str(burst_in.burst_id).upper().replace('_', '-') + + # create substring "{end_date}" + end_date_str = burst_in.sensing_stop.strftime('%Y-%m-%d') + # source data access (URL or DOI) source_data_access = cfg_in.groups.input_file_group.source_data_access if not source_data_access: @@ -424,12 +431,20 @@ def get_metadata_dict(product_id: str, product_data_access = cfg_in.groups.product_group.product_data_access if not product_data_access: product_data_access = '(NOT PROVIDED)' + else: + # replace "{burst_id}"" and "{end_date}" substrings + product_data_access = product_data_access.format( + burst_id=burst_id_asf, end_date=end_date_str) # static layers data access (URL or DOI) static_layers_data_access = \ cfg_in.groups.product_group.static_layers_data_access if not static_layers_data_access: static_layers_data_access = '(NOT PROVIDED)' + else: + # replace "{burst_id}"" and "{end_date}" substrings + static_layers_data_access = static_layers_data_access.format( + burst_id=burst_id_asf, end_date=end_date_str) # platform ID if burst_in.platform_id == 'S1A': diff --git a/src/rtc/mosaic_geobursts.py b/src/rtc/mosaic_geobursts.py index b47aac6c..f402f92f 100644 --- a/src/rtc/mosaic_geobursts.py +++ b/src/rtc/mosaic_geobursts.py @@ -623,13 +623,13 @@ def mosaic_single_output_file(list_rtc_images, list_nlooks, mosaic_filename, for i_band in range(num_bands): gdal_band = raster_out.GetRasterBand(i_band+1) - gdal_band.WriteArray(arr_numerator[i_band]) gdal_band.SetDescription(description_list[i_band]) if ctable is not None: gdal_band.SetRasterColorTable(ctable) gdal_band.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) if no_data_value is not None: gdal_band.SetNoDataValue(no_data_value) + gdal_band.WriteArray(arr_numerator[i_band]) band_ds = None reference_raster = None @@ -710,12 +710,12 @@ def mosaic_multiple_output_files( # for i_band in range(num_bands): band_out = raster_out.GetRasterBand(1) - band_out.WriteArray(arr_numerator[i_band]) if ctable is not None: band_out.SetRasterColorTable(ctable) band_out.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) if no_data_value is not None: band_out.SetNoDataValue(no_data_value) + band_out.WriteArray(arr_numerator[i_band]) band_ds = None reference_raster = None diff --git a/src/rtc/rtc_s1_single_job.py b/src/rtc/rtc_s1_single_job.py index 9e695037..098514f1 100755 --- a/src/rtc/rtc_s1_single_job.py +++ b/src/rtc/rtc_s1_single_job.py @@ -822,13 +822,13 @@ def set_mask_fill_value_and_ctable(mask_file, reference_file): mask_gdal_band = mask_gdal_ds.GetRasterBand(1) mask_array = mask_gdal_band.ReadAsArray() mask_array[(np.isnan(ref_array)) & (mask_array == 0)] = 255 - mask_gdal_band.WriteArray(mask_array) mask_gdal_band.SetNoDataValue(255) mask_ctable.SetColorEntry(255, (0, 0, 0, 0)) mask_gdal_band.SetRasterColorTable(mask_ctable) mask_gdal_band.SetRasterColorInterpretation( gdal.GCI_PaletteIndex) + mask_gdal_band.WriteArray(mask_array) del mask_gdal_band del mask_gdal_ds diff --git a/src/rtc/version.py b/src/rtc/version.py index dc0bfb4b..e506e440 100644 --- a/src/rtc/version.py +++ b/src/rtc/version.py @@ -1 +1 @@ -VERSION='1.0.1' +VERSION='1.0.2'