From 37aed0104780d18a6ef76adbe38d9caebc8221c9 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Sat, 9 Sep 2023 15:14:38 -0700 Subject: [PATCH 01/10] product version can be str or num --- src/rtc/schemas/rtc_s1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtc/schemas/rtc_s1.yaml b/src/rtc/schemas/rtc_s1.yaml index 7175a75..9c2798d 100644 --- a/src/rtc/schemas/rtc_s1.yaml +++ b/src/rtc/schemas/rtc_s1.yaml @@ -40,7 +40,7 @@ runconfig: processing_type: enum('NOMINAL', 'URGENT', 'CUSTOM', 'UNDEFINED', required=False) - product_version: str(required=False) + product_version: any(str(), num(), required=False) # Directory where PGE will place results product_path: str() From 3920cf668f7a4ff363b9539ea0383e9a1b58315b Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 00:17:54 -0700 Subject: [PATCH 02/10] update HDF5 metadata data types --- src/rtc/h5_prep.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 617a122..fa6d5b9 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -440,19 +440,38 @@ def get_metadata_dict(product_id: str, error_msg = f'ERROR Not recognized platform ID: {burst_in.platform_id}' raise NotImplementedError(error_msg) + # geocoding algorithm + geocoding_algorithm = cfg_in.groups.processing.geocoding.algorithm_type + if geocoding_algorithm == 'area_projection': + geocoding_algorithm = ('Area-Based SAR Geocoding with Adaptive' + ' Multilooking (GEO-AP)') + + # RTC algorithm + rtc_algorithm = cfg_in.groups.processing.rtc.algorithm_type + if rtc_algorithm == 'area_projection': + rtc_algorithm = 'Area-Based SAR Radiometric Terrain Correction (RTC-AP)' + # burst and mosaic snap values burst_snap_x = cfg_in.groups.processing.geocoding.bursts_geogrid.x_snap if not burst_snap_x: burst_snap_x = '(DISABLED)' + else: + burst_snap_x = float(burst_snap_x) burst_snap_y = cfg_in.groups.processing.geocoding.bursts_geogrid.y_snap if not burst_snap_y: burst_snap_y = '(DISABLED)' + else: + burst_snap_y = float(burst_snap_y) mosaic_snap_x = cfg_in.groups.processing.mosaicking.mosaic_geogrid.x_snap if not mosaic_snap_x: mosaic_snap_x = '(DISABLED)' + else: + mosaic_snap_x = float(mosaic_snap_x) mosaic_snap_y = cfg_in.groups.processing.mosaicking.mosaic_geogrid.y_snap if not mosaic_snap_y: mosaic_snap_y = '(DISABLED)' + else: + mosaic_snap_y = float(mosaic_snap_y) # Geometric accuracy estimated_geometric_accuracy_bias_y = \ @@ -547,7 +566,7 @@ def get_metadata_dict(product_id: str, 'identification/productSpecificationVersion': ['product_specification_version', ALL_PRODUCTS, - PRODUCT_SPECIFICATION_VERSION, + str(PRODUCT_SPECIFICATION_VERSION), 'Product specification version which represents the schema of' ' this product'], 'identification/acquisitionMode': @@ -574,12 +593,6 @@ def get_metadata_dict(product_id: str, # ['is_urgent_observation', False, # 'List of booleans indicating if datatakes # are nominal or urgent'], - 'identification/diagnosticModeFlag': - ['diagnostic_mode_flag', - STANDARD_RTC_S1_ONLY, - False, - 'Indicates if the radar mode is a diagnostic mode or not: True or' - ' False'], 'identification/isGeocoded': [None, ALL_PRODUCTS, @@ -670,7 +683,7 @@ def get_metadata_dict(product_id: str, 'metadata/sourceData/rangeBandwidth': ['source_data_range_bandwidth', STANDARD_RTC_S1_ONLY, - str(burst_in.range_bandwidth), + burst_in.range_bandwidth, 'Processed range bandwidth in Hz'], # populate source data processingDateTime with from processing_info @@ -764,7 +777,7 @@ def get_metadata_dict(product_id: str, 'metadata/sourceData/maxNoiseEquivalentSigmaZero': # 1.6.9 [None, STANDARD_RTC_S1_ONLY, - -22, + -22.0, 'Maximum Noise equivalent sigma0 in dB'], 'metadata/processingInformation/algorithms/softwareVersion': @@ -942,14 +955,14 @@ def get_metadata_dict(product_id: str, ['processing_information' '_geocoding_algorithm', ALL_PRODUCTS, - cfg_in.groups.processing.geocoding.algorithm_type, + geocoding_algorithm, 'Geocoding algorithm'], 'metadata/processingInformation/algorithms/' + 'radiometricTerrainCorrection': ['processing_information' '_radiometric_terrain_correction_algorithm', ALL_PRODUCTS, - cfg_in.groups.processing.rtc.algorithm_type, + rtc_algorithm, 'Radiometric terrain correction (RTC) algorithm'], 'metadata/processingInformation/algorithms/isce3Version': ['isce3_version', From 346176e81e7248df053346c1b83cce5963487fde Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 08:42:38 -0700 Subject: [PATCH 03/10] remove az/rg resolution in meters from RTC-S1-STATIC metadata; substitute underbars with dashes in product type --- src/rtc/h5_prep.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index fa6d5b9..1c0e2c6 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -384,6 +384,11 @@ def get_metadata_dict(product_id: str, the field is not saved on the GeoTIFF file ''' + # product type. Update "RTC_S1" and "RTC_S1_STATIC" to + # "RTC_S1" and "RTC_S1_STATIC", respectively. + product_type = \ + cfg_in.groups.primary_executable.product_type.replace('_', '-') + # orbit files orbit_files = [os.path.basename(f) for f in cfg_in.orbit_path] @@ -539,7 +544,7 @@ def get_metadata_dict(product_id: str, 'identification/productType': ['product_type', ALL_PRODUCTS, - cfg_in.groups.primary_executable.product_type, + product_type, 'Product type'], 'identification/project': ['project', @@ -743,12 +748,12 @@ def get_metadata_dict(product_id: str, ' of the source data'], 'metadata/sourceData/azimuthResolutionInMeters': # 1.6.7 ['source_data_azimuth_resolution_in_meters', - ALL_PRODUCTS, + STANDARD_RTC_S1_ONLY, azimuth_resolution, 'Azimuth resolution of the source data in meters'], 'metadata/sourceData/slantRangeResolutionInMeters': # 1.6.7 ['source_data_slant_range_resolution_in_meters', - ALL_PRODUCTS, + STANDARD_RTC_S1_ONLY, slant_range_resolution, 'Slant-range resolution of the source data in meters'], From dbb4f49b22eadf9e537899077e8a58fbe67e92df Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 09:26:35 -0700 Subject: [PATCH 04/10] move the metadata field ceosAnalysisReadyDataPixelCoordinateConvention as an attribute of boundingBox --- src/rtc/h5_prep.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 1c0e2c6..ae3fabf 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -454,7 +454,8 @@ def get_metadata_dict(product_id: str, # RTC algorithm rtc_algorithm = cfg_in.groups.processing.rtc.algorithm_type if rtc_algorithm == 'area_projection': - rtc_algorithm = 'Area-Based SAR Radiometric Terrain Correction (RTC-AP)' + rtc_algorithm = ('Area-Based SAR Radiometric Terrain Correction' + ' (RTC-AP)') # burst and mosaic snap values burst_snap_x = cfg_in.groups.processing.geocoding.bursts_geogrid.x_snap @@ -879,15 +880,6 @@ def get_metadata_dict(product_id: str, 'backscatter_dB = 10*log10(backscatter_linear)', 'Equation to convert provided backscatter to decibel (dB)'], - # 1.7.8 - ('metadata/processingInformation/parameters/geocoding/' - '/ceosAnalysisReadyDataPixelCoordinateConvention'): - ['processing_information_' - 'ceos_analysis_ready_data_pixel_coordinate_convention', - ALL_PRODUCTS, - 'ULC', - 'CEOS Analysis Ready Data (CARD) pixel coordinate convention'], - # 4.4 ('metadata/processingInformation/parameters/geocoding/' 'burstGeogridSnapX'): @@ -1117,18 +1109,26 @@ def get_metadata_dict(product_id: str, ] metadata_dict['identification/boundingBox'] = \ - [None, + ['bounding_box', ALL_PRODUCTS, np.array(xy_bounding_box), # 1.7.5 'Bounding box of the product, in order of xmin, ymin, xmax, ymax'] # Attribute `epsg` for HDF5 dataset /identification/boundingBox metadata_dict['identification/boundingBox[epsg]'] = \ - [None, + ['bounding_box_epsg_code', ALL_PRODUCTS, str(epsg_geogrid), 'Bounding box EPSG code'] + # 1.7.8 + metadata_dict['identification/boundingBox' + '[pixel_coordinate_convention]'] = \ + ['bounding_box_pixel_coordinate_convention', + ALL_PRODUCTS, + 'Upper left corner (ULC)', + 'Bounding box pixel coordinate convention'] + metadata_dict['identification/burstID'] = \ ['burst_id', ALL_PRODUCTS, From 5263489d746614c07ba5485813f87c1f9f1fe946 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 09:41:33 -0700 Subject: [PATCH 05/10] move the metadata field ceosAnalysisReadyDataPixelCoordinateConvention as an attribute of boundingBox (2) --- src/rtc/h5_prep.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index ae3fabf..08fa37d 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -1108,10 +1108,11 @@ def get_metadata_dict(product_id: str, ymax_geogrid ] + # 1.7.5 metadata_dict['identification/boundingBox'] = \ ['bounding_box', ALL_PRODUCTS, - np.array(xy_bounding_box), # 1.7.5 + xy_bounding_box, 'Bounding box of the product, in order of xmin, ymin, xmax, ymax'] # Attribute `epsg` for HDF5 dataset /identification/boundingBox From 85c3e0fc5815edd79ca4db9b872634772cda4283 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 11:18:25 -0700 Subject: [PATCH 06/10] geometric accuracy fields are float --- src/rtc/h5_prep.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 08fa37d..b8ef004 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -491,12 +491,24 @@ def get_metadata_dict(product_id: str, if not estimated_geometric_accuracy_bias_y: estimated_geometric_accuracy_bias_y = '(UNSPECIFIED)' + else: + estimated_geometric_accuracy_bias_y = float( + estimated_geometric_accuracy_bias_y) if not estimated_geometric_accuracy_bias_x: estimated_geometric_accuracy_bias_x = '(UNSPECIFIED)' + else: + estimated_geometric_accuracy_bias_x = float( + estimated_geometric_accuracy_bias_x) if not estimated_geometric_accuracy_stddev_y: estimated_geometric_accuracy_stddev_y = '(UNSPECIFIED)' + else: + estimated_geometric_accuracy_stddev_y = float( + estimated_geometric_accuracy_stddev_y) if not estimated_geometric_accuracy_stddev_x: estimated_geometric_accuracy_stddev_x = '(UNSPECIFIED)' + else: + estimated_geometric_accuracy_stddev_x = float( + estimated_geometric_accuracy_stddev_x) subswath_id = burst_in.swath_name.upper() From 3935407441e31a1155a1b9da65ecf02c9dbbd759 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 11:20:40 -0700 Subject: [PATCH 07/10] remove commented code --- src/rtc/h5_prep.py | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index b8ef004..485d88f 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -531,16 +531,11 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, burst_in.abs_orbit_number, 'Absolute orbit number'], - # NOTE: The field below does not exist on opera_rtc.xml - # 'identification/relativeOrbitNumber': - # [int(burst_in.burst_id[1:4]), 'Relative orbit number'], 'identification/trackNumber': ['track_number', ALL_PRODUCTS, burst_in.burst_id.track_number, 'Track number'], - # 'identification/missionId': - # [mission_id, 'Mission identifier'], 'identification/platform': ['platform', ALL_PRODUCTS, @@ -607,10 +602,6 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, burst_in.orbit_direction.lower(), 'Orbit direction can be ascending or descending'], - # 'identification/isUrgentObservation': - # ['is_urgent_observation', False, - # 'List of booleans indicating if datatakes - # are nominal or urgent'], 'identification/isGeocoded': [None, ALL_PRODUCTS, @@ -680,11 +671,7 @@ def get_metadata_dict(product_id: str, source_data_access, 'Location from where the source data can be retrieved' ' (URL or DOI)'], - # 'metadata/sourceData/radarBand': # 1.6.4 - # ['radar_band', 'C', 'Acquired frequency band'], - # TODO: review, should it be - # burst_in.burst_misc_metadata.processing_info_dict["organisation"]? 'metadata/sourceData/institution': ['source_data_institution', ALL_PRODUCTS, @@ -704,8 +691,6 @@ def get_metadata_dict(product_id: str, burst_in.range_bandwidth, 'Processed range bandwidth in Hz'], - # populate source data processingDateTime with from processing_info - # "stop" (SLC Post processing date time) 'metadata/sourceData/processingDateTime': # 1.6.6 ['source_data_processing_datetime', ALL_PRODUCTS, @@ -744,10 +729,6 @@ def get_metadata_dict(product_id: str, STANDARD_RTC_S1_ONLY, burst_in.radar_center_frequency, 'Center frequency of the processed image in Hz'], - # 'metadata/sourceData/geometry': # 1.6.7 - # ['source_data_geometry', - # 'slant range', - # 'Geometry of the source data'], 'metadata/sourceData/zeroDopplerTimeSpacing': # 1.6.7 ['source_data_zero_doppler_time_spacing', ALL_PRODUCTS, @@ -770,15 +751,6 @@ def get_metadata_dict(product_id: str, slant_range_resolution, 'Slant-range resolution of the source data in meters'], - # 'metadata/sourceData/azimuthResolution': # 1.6.7 - # ['source_azimuth_resolution', - # burst_in.azimuth_time_interval, - # 'Azimuth time resolution of the source data in seconds'], - # 'metadata/sourceData/slantRangeResolution': # 1.6.7 - # ['source_slant_range_resolution', - # burst_in.range_pixel_spacing, - # 'Slant range resolution of the source data in meters'], - 'metadata/sourceData/nearRangeIncidenceAngle': # 1.6.7 [None, STANDARD_RTC_S1_ONLY, @@ -936,19 +908,7 @@ def get_metadata_dict(product_id: str, ('An estimate of the localization error standard deviation' ' in the easting direction')], - # 'identification/frameNumber': # TBD - # 'identification/plannedDatatakeId': - # 'identification/plannedObservationId': - - # f'{FREQ_GRID_DS}/faradayRotationFlag': - # ['faraday_rotation_flag', False, - # 'Flag to indicate if Faraday Rotation correction was applied'], - # f'{FREQ_GRID_DS}/polarizationOrientationFlag': - # ['polarization_orientation_flag', False, - # 'Flag to indicate if Polarization Orientation correction was - # applied'], - - 'metadata/processingInformation/algorithms/demInterpolation': + 'metadata/processingInformation/algorithms/demInterpolation': ['processing_information' '_dem_interpolation_algorithm', ALL_PRODUCTS, @@ -978,9 +938,6 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, isce3.__version__, 'Version of the ISCE3 framework used for processing'], - # 'metadata/processingInformation/algorithms/RTCVersion': - # [str(SOFTWARE_VERSION), - # 'RTC-S1 SAS version used for processing'], 'metadata/processingInformation/algorithms/s1ReaderVersion': ['s1_reader_version', ALL_PRODUCTS, From 3122e0c964c727744615866dbdc6c2dd264c65de Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 11:23:17 -0700 Subject: [PATCH 08/10] lint code --- src/rtc/h5_prep.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 485d88f..ecf47d2 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -908,7 +908,7 @@ def get_metadata_dict(product_id: str, ('An estimate of the localization error standard deviation' ' in the easting direction')], - 'metadata/processingInformation/algorithms/demInterpolation': + 'metadata/processingInformation/algorithms/demInterpolation': ['processing_information' '_dem_interpolation_algorithm', ALL_PRODUCTS, @@ -1116,7 +1116,8 @@ def get_metadata_dict(product_id: str, ['zero_doppler_start_time', ALL_PRODUCTS, burst_in.sensing_start.strftime(DATE_TIME_METADATA_FORMAT), - 'Azimuth start time of the product in the format YYYY-MM-DDThh:mm:ss.sZ'] # 1.6.3 + 'Azimuth start time of the product in the format' + ' YYYY-MM-DDThh:mm:ss.sZ'] # 1.6.3 # TODO: Update for static layers!!! metadata_dict['identification/zeroDopplerEndTime'] = \ @@ -1131,7 +1132,7 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, burst_in.sensing_start.strftime(DATE_TIME_METADATA_FORMAT), 'Azimuth start time of the input product in the format' - ' YYYY-MM-DDThh:mm:ss.sZ'] # 1.6.3 + ' YYYY-MM-DDThh:mm:ss.sZ'] # 1.6.3 metadata_dict['metadata/sourceData/zeroDopplerEndTime'] = \ ['source_data_zero_doppler_end_time', ALL_PRODUCTS, @@ -1403,7 +1404,8 @@ def get_rfi_metadata_dict(burst_in, rfi_root_path): rfi_metadata_dict[f'{rfi_root_path}/isRfiInfoAvailable'] =\ ['qa_rfi_info_available', not is_rfi_info_empty, - 'A flag to indicate whether RFI information is available in the source data'] + 'A flag to indicate whether RFI information is available in the' + ' source data'] if is_rfi_info_empty: return rfi_metadata_dict @@ -1426,8 +1428,8 @@ def get_rfi_metadata_dict(burst_in, rfi_root_path): ['qa_rfi_burst_report_azimuth_time', burst_in.burst_rfi_info.rfi_burst_report['azimuthTime'].strftime( DATE_TIME_METADATA_FORMAT), - 'Sensing start time of the burst that corresponds to the RFI report' - ' in the format YYYY-MM-DDThh:mm:ss.sZ'], + 'Sensing start time of the burst that corresponds to the RFI' + ' report in the format YYYY-MM-DDThh:mm:ss.sZ'], 'rfiBurstReport/inBandOutBandPowerRatio': ['qa_rfi_in_band_out_band_power_ratio', burst_in.burst_rfi_info.rfi_burst_report[ From e4b4d37e054176227cf842b71bd47ee2a9bf3e03 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 11:48:54 -0700 Subject: [PATCH 09/10] fix product_type --- src/rtc/h5_prep.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index ecf47d2..caa9951 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -384,11 +384,6 @@ def get_metadata_dict(product_id: str, the field is not saved on the GeoTIFF file ''' - # product type. Update "RTC_S1" and "RTC_S1_STATIC" to - # "RTC_S1" and "RTC_S1_STATIC", respectively. - product_type = \ - cfg_in.groups.primary_executable.product_type.replace('_', '-') - # orbit files orbit_files = [os.path.basename(f) for f in cfg_in.orbit_path] @@ -401,6 +396,10 @@ def get_metadata_dict(product_id: str, # product type product_type = cfg_in.groups.primary_executable.product_type + # product type. Update "RTC_S1" and "RTC_S1_STATIC" to + # "RTC_S1" and "RTC_S1_STATIC", respectively. + product_type_metadata_value = product_type.replace('_', '-') + # product version product_version_runconfig = cfg_in.groups.product_group.product_version product_version = get_product_version(product_version_runconfig) @@ -552,7 +551,7 @@ def get_metadata_dict(product_id: str, 'identification/productType': ['product_type', ALL_PRODUCTS, - product_type, + product_type_metadata_value, 'Product type'], 'identification/project': ['project', From f3395509d5b00d07f9de506a9fe29ef4d4941754 Mon Sep 17 00:00:00 2001 From: "Gustavo H. X. Shiroma" Date: Mon, 11 Sep 2023 12:41:52 -0700 Subject: [PATCH 10/10] fix assignment of the GeoTIFF metadata to the correct (burst) output file list --- src/rtc/rtc_s1.py | 22 +++++++++++++--------- src/rtc/rtc_s1_single_job.py | 24 ++++++++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/rtc/rtc_s1.py b/src/rtc/rtc_s1.py index 7be98e6..78f6384 100755 --- a/src/rtc/rtc_s1.py +++ b/src/rtc/rtc_s1.py @@ -615,6 +615,8 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): logger.info(f'Processing burst: {burst_id} ({burst_index+1}/' f'{n_bursts})') + burst_output_file_list = [] + pol_list = list(burst_pol_dict.keys()) burst = burst_pol_dict[pol_list[0]] geogrid = cfg.geogrids[burst_id] @@ -683,7 +685,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(nlooks_file) else: - output_file_list.append(nlooks_file) + burst_output_file_list.append(nlooks_file) else: nlooks_file = None @@ -700,7 +702,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(rtc_anf_file) else: - output_file_list.append(rtc_anf_file) + burst_output_file_list.append(rtc_anf_file) else: rtc_anf_file = None @@ -720,7 +722,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(rtc_anf_gamma0_to_sigma0_file) else: - output_file_list.append(rtc_anf_gamma0_to_sigma0_file) + burst_output_file_list.append(rtc_anf_gamma0_to_sigma0_file) else: rtc_anf_gamma0_to_sigma0_file = None @@ -746,7 +748,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): temp_files_list.append(layover_shadow_mask_file) layover_shadow_mask_file = None else: - output_file_list.append(layover_shadow_mask_file) + burst_output_file_list.append(layover_shadow_mask_file) logger.info(f'file saved: {layover_shadow_mask_file}') # Take the layover shadow mask from HDF5 file if not exists @@ -796,7 +798,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): temp_files_list.append(geo_burst_vrt_filename) if not flag_bursts_files_are_temporary: - output_file_list += output_burst_imagery_list + burst_output_file_list += output_burst_imagery_list else: temp_files_list += output_burst_imagery_list @@ -834,14 +836,14 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(current_file) else: - output_file_list.append(current_file) + burst_output_file_list.append(current_file) # Browse image (burst) using static layers if (flag_save_browse and product_type == STATIC_LAYERS_PRODUCT_TYPE): browse_image_filename = \ os.path.join(output_dir_bursts, f'{burst_product_id}.png') - output_file_list.append(browse_image_filename) + burst_output_file_list.append(browse_image_filename) # Create burst HDF5 if (save_hdf5_metadata and save_bursts): @@ -850,7 +852,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): output_hdf5_file_burst = os.path.join( hdf5_file_output_dir, f'{burst_product_id}.{hdf5_file_extension}') - output_file_list.append(output_hdf5_file_burst) + burst_output_file_list.append(output_hdf5_file_burst) # Create mosaic HDF5 if (save_hdf5_metadata and save_mosaics @@ -864,7 +866,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): product_type != STATIC_LAYERS_PRODUCT_TYPE): browse_image_filename = \ os.path.join(output_dir_bursts, f'{burst_product_id}.png') - output_file_list.append(browse_image_filename) + burst_output_file_list.append(browse_image_filename) # Save mosaic metadata for later use if (save_mosaics and burst_index == 0): @@ -875,6 +877,8 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): all_metadata_dict_to_geotiff_metadata_dict( mosaic_metadata_dict) + output_file_list += burst_output_file_list + t_burst_end = time.time() logger.info( f'elapsed time (burst): {t_burst_end - t_burst_start}') diff --git a/src/rtc/rtc_s1_single_job.py b/src/rtc/rtc_s1_single_job.py index 261a03e..9e69503 100755 --- a/src/rtc/rtc_s1_single_job.py +++ b/src/rtc/rtc_s1_single_job.py @@ -1443,6 +1443,8 @@ def run_single_job(cfg: RunConfig): logger.info(f'Processing burst: {burst_id} ({burst_index+1}/' f'{n_bursts})') + burst_output_file_list = [] + geogrid = cfg.geogrids[burst_id] pol_list = list(burst_pol_dict.keys()) burst = burst_pol_dict[pol_list[0]] @@ -1586,7 +1588,7 @@ def run_single_job(cfg: RunConfig): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(nlooks_file) else: - output_file_list.append(nlooks_file) + burst_output_file_list.append(nlooks_file) else: nlooks_file = None @@ -1598,7 +1600,7 @@ def run_single_job(cfg: RunConfig): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(rtc_anf_file) else: - output_file_list.append(rtc_anf_file) + burst_output_file_list.append(rtc_anf_file) else: rtc_anf_file = None @@ -1612,7 +1614,7 @@ def run_single_job(cfg: RunConfig): if flag_bursts_secondary_files_are_temporary: temp_files_list.append(rtc_anf_gamma0_to_sigma0_file) else: - output_file_list.append(rtc_anf_gamma0_to_sigma0_file) + burst_output_file_list.append(rtc_anf_gamma0_to_sigma0_file) else: rtc_anf_gamma0_to_sigma0_file = None @@ -1715,7 +1717,7 @@ def run_single_job(cfg: RunConfig): if flag_layover_shadow_mask_is_temporary: temp_files_list.append(layover_shadow_mask_file) else: - output_file_list.append(layover_shadow_mask_file) + burst_output_file_list.append(layover_shadow_mask_file) logger.info(f'file saved: {layover_shadow_mask_file}') if save_mask: output_metadata_dict[LAYER_NAME_LAYOVER_SHADOW_MASK][1].append( @@ -1858,7 +1860,7 @@ def run_single_job(cfg: RunConfig): output_burst_imagery_list, output_raster_format, logger) - output_file_list += output_burst_imagery_list + burst_output_file_list += output_burst_imagery_list if save_nlooks: if flag_process: @@ -1913,7 +1915,7 @@ def run_single_job(cfg: RunConfig): # files are temporary temp_files_list += radar_grid_file_dict_filenames else: - output_file_list += radar_grid_file_dict_filenames + burst_output_file_list += radar_grid_file_dict_filenames # Save browse image (burst) using static layers if (flag_save_browse and @@ -1925,7 +1927,7 @@ def run_single_job(cfg: RunConfig): browse_image_burst_height, browse_image_burst_width, temp_files_list, burst_scratch_path, logger) - output_file_list.append(browse_image_filename) + burst_output_file_list.append(browse_image_filename) # Create burst HDF5 if (flag_process and save_hdf5_metadata and save_bursts): @@ -1943,7 +1945,7 @@ def run_single_job(cfg: RunConfig): layover_shadow_mask_file, radar_grid_file_dict, save_imagery=save_imagery_as_hdf5, save_secondary_layers=save_secondary_layers_as_hdf5) - output_file_list.append(output_hdf5_file_burst) + burst_output_file_list.append(output_hdf5_file_burst) # Save browse image (burst) if (flag_process and flag_save_browse and @@ -1955,7 +1957,7 @@ def run_single_job(cfg: RunConfig): pol_list, browse_image_burst_height, browse_image_burst_width, temp_files_list, burst_scratch_path, logger) - output_file_list.append(browse_image_filename) + burst_output_file_list.append(browse_image_filename) # Append metadata to burst GeoTIFFs if (flag_process and (not flag_bursts_files_are_temporary or @@ -1965,7 +1967,7 @@ def run_single_job(cfg: RunConfig): is_mosaic=False) geotiff_metadata_dict = all_metadata_dict_to_geotiff_metadata_dict( metadata_dict) - for current_file in output_file_list: + for current_file in burst_output_file_list: if not current_file.endswith('.tif'): continue append_metadata_to_geotiff_file(current_file, @@ -1988,6 +1990,8 @@ def run_single_job(cfg: RunConfig): all_metadata_dict_to_geotiff_metadata_dict( mosaic_metadata_dict) + output_file_list += burst_output_file_list + t_burst_end = time.time() logger.info( f'elapsed time (burst): {t_burst_end - t_burst_start}')