diff --git a/setup.py b/setup.py index c5067785..5d3a90d4 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def _get_version(): package_data_dict['rtc'] = [ os.path.join('defaults', 'rtc_s1.yaml'), - os.path.join('defaults', 'rtc_s1_sl.yaml'), + os.path.join('defaults', 'rtc_s1_static.yaml'), os.path.join('schemas', 'rtc_s1.yaml')] setup( diff --git a/src/rtc/defaults/rtc_s1.yaml b/src/rtc/defaults/rtc_s1.yaml index 1ba13ee3..ae1d2735 100644 --- a/src/rtc/defaults/rtc_s1.yaml +++ b/src/rtc/defaults/rtc_s1.yaml @@ -3,7 +3,7 @@ runconfig: groups: - # Required. Output product type: "RTC_S1" or "RTC_S1_SL" + # Required. Output product type: "RTC_S1" or "RTC_S1_STATIC" primary_executable: product_type: RTC_S1 @@ -13,8 +13,13 @@ runconfig: input_file_group: # Required. List of SAFE files (min=1) safe_file_path: + + # Location from where the source data can be retrieved (URL or DOI) + source_data_access: + # Required. List of orbit (EOF) files (min=1) orbit_file_path: + # Optional. Burst ID to process (empty for all bursts) burst_id: @@ -42,9 +47,9 @@ runconfig: scratch_path: # If option `save_bursts` is set, output bursts are saved to: - # {output_dir}/{burst_id}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} # If option `save_mosaics` is set, output mosaics are saved to: - # {output_dir}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{product_id}{suffix}.{ext} # # If the `product_id` contains the substring "_{burst_id}", the # substring will be substituted by either: @@ -55,7 +60,7 @@ runconfig: # `RTC-S1_T069-147170-IW1_S1B` for the burst t069-147170-IW1; and it # will become `RTC-S1_S1B` for the mosaic product. # - # If the field `product_id` is left empty, the product ID will + # If the field `product_id` is left empty, the burst product ID will # follow the RTC-S1 file naming conventions: # `OPERA_L2_RTC-S1_{burst_id}_{sensing_start_datetime}_ # {processing_datetime}_{sensor}_{pixel_spacing} @@ -66,6 +71,12 @@ runconfig: output_dir: product_id: + # Validity start date for RTC-S1-STATIC products in the format YYYYMMDD + rtc_s1_static_validity_start_date: + + # Location from where the output product can be retrieved (URL or DOI) + product_data_access: + # Save RTC-S1 bursts save_bursts: True @@ -224,7 +235,7 @@ runconfig: # Layover/shadow mask dilation size of shadow pixels # (values 1 and 3) - shadow_dilation_size: 3 + shadow_dilation_size: 0 # OPTIONAL - Absolute radiometric correction abs_rad_cal: 1 diff --git a/src/rtc/defaults/rtc_s1_sl.yaml b/src/rtc/defaults/rtc_s1_static.yaml similarity index 89% rename from src/rtc/defaults/rtc_s1_sl.yaml rename to src/rtc/defaults/rtc_s1_static.yaml index fe799645..8b385fc3 100644 --- a/src/rtc/defaults/rtc_s1_sl.yaml +++ b/src/rtc/defaults/rtc_s1_static.yaml @@ -5,8 +5,8 @@ runconfig: primary_executable: - # Required. Output product type: "RTC_S1" or "RTC_S1_SL" - product_type: RTC_S1_SL + # Required. Output product type: "RTC_S1" or "RTC_S1_STATIC" + product_type: RTC_S1_STATIC pge_name_group: pge_name: RTC_S1_PGE @@ -14,8 +14,13 @@ runconfig: input_file_group: # Required. List of SAFE files (min=1) safe_file_path: + + # Location from where the source data can be retrieved (URL or DOI) + source_data_access: + # Required. List of orbit (EOF) files (min=1) orbit_file_path: + # Optional. Burst ID to process (empty for all bursts) burst_id: @@ -43,26 +48,36 @@ runconfig: scratch_path: # If option `save_bursts` is set, output bursts are saved to: - # {output_dir}/{burst_id}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} # If option `save_mosaics` is set, output mosaics are saved to: - # {output_dir}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{product_id}{suffix}.{ext} # # If the `product_id` contains the substring "_{burst_id}", the # substring will be substituted by either: # - "_" followed by the burst ID, if the product is a burst; or # - An empty string, if the product is a mosaic. # - # For example, the `product_id` = `RTC-S1_{burst_id}_S1B` will become - # `RTC-S1_069-147170-IW1_S1B` for the burst t069-147170-IW1; and it - # will become `RTC-S1_S1B` for the mosaic product. + # For example, the `product_id` = `RTC-S1-STATIC_{burst_id}_S1B` will become + # `RTC-S1-STATIC_069-147170-IW1_S1B` for the burst t069-147170-IW1; and it + # will become `RTC-S1-STATIC_S1B` for the mosaic product. + # + # If the field `product_id` is left empty, the burst product ID will + # follow the RTC-S1-STATIC file naming conventions: + # `OPERA_L2_RTC-S1-STATIC_{burst_id}_{rtc_s1_static_validity_start_date}_ + # {processing_datetime}_{sensor}_{pixel_spacing} + # _{product_version}`. # - # If the field `product_id`` is left empty, the prefix - # "OPERA_L2_RTC-S1_{burst_id}" will be used instead. # `suffix` is only used when there are multiple output files. # `ext` is determined by geocoding_options.output_imagery_format. output_dir: product_id: + # Validity start date for RTC-S1-STATIC products in the format YYYYMMDD + rtc_s1_static_validity_start_date: + + # Location from where the output product can be retrieved (URL or DOI) + product_data_access: + # Save RTC-S1 bursts save_bursts: True @@ -221,7 +236,7 @@ runconfig: # Layover/shadow mask dilation size of shadow pixels # (values 1 and 3) - shadow_dilation_size: 3 + shadow_dilation_size: 0 # OPTIONAL - Absolute radiometric correction abs_rad_cal: 1 diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 2f578764..7027cc34 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -40,7 +40,7 @@ LAYER_NAME_PROJECTION_ANGLE = 'projection_angle' LAYER_NAME_RTC_ANF_PROJECTION_ANGLE = 'rtc_anf_projection_angle' LAYER_NAME_RANGE_SLOPE = 'range_slope' -LAYER_NAME_DEM = 'dem' +LAYER_NAME_DEM = 'interpolated_dem' # RTC-S1 product layer names layer_hdf5_dict = { @@ -374,6 +374,17 @@ def get_metadata_dict(product_id: str, # If the DEM description is not provided, use DEM source dem_file_description = os.path.basename(cfg_in.dem) + # source data access (URL or DOI) + source_data_access = cfg_in.groups.input_file_group.source_data_access + if not source_data_access: + source_data_access = '(NOT PROVIDED)' + + # product data access (URL or DOI) + product_data_access = cfg_in.groups.product_group.product_data_access + if not product_data_access: + product_data_access = '(NOT PROVIDED)' + + # platform ID if burst_in.platform_id == 'S1A': platform_id = 'Sentinel-1A' elif burst_in.platform_id == 'S1B': @@ -386,6 +397,20 @@ def get_metadata_dict(product_id: str, error_msg = f'ERROR Not recognized platform ID: {burst_in.platform_id}' raise NotImplementedError(error_msg) + # burst and mosaic snap values + burst_snap_x = cfg_in.groups.processing.geocoding.bursts_geogrid.x_snap + if not burst_snap_x: + burst_snap_x = '(DISABLED)' + burst_snap_y = cfg_in.groups.processing.geocoding.bursts_geogrid.y_snap + if not burst_snap_y: + burst_snap_y = '(DISABLED)' + mosaic_snap_x = cfg_in.groups.processing.mosaicking.mosaic_geogrid.x_snap + if not mosaic_snap_x: + mosaic_snap_x = '(DISABLED)' + mosaic_snap_y = cfg_in.groups.processing.mosaicking.mosaic_geogrid.y_snap + if not mosaic_snap_y: + mosaic_snap_y = '(DISABLED)' + # mission_id = 'Sentinel' # Manifests the field names, corresponding values from RTC workflow, and @@ -464,10 +489,11 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, 'Interferometric Wide (IW)', 'Acquisition mode'], - # 'identification/CARDProductType': - # ['card_product_type', 'Normalised Radar Backscatter', - # 'CARD Product type'], # 1.3 - + 'identification/ceosAnalysisReadyDataProductType': # 1.3 + ['ceos_analysis_ready_data_product_type', + ALL_PRODUCTS, + 'Normalised Radar Backscatter', + 'CEOS Analysis Ready Data (CARD) product type'], 'identification/lookDirection': ['look_direction', ALL_PRODUCTS, @@ -514,7 +540,7 @@ def get_metadata_dict(product_id: str, processing_type, 'Processing type: "NOMINAL", "URGENT", "CUSTOM", or "UNDEFINED"'], 'identification/processingDateTime': - ['processing_date_time', + ['processing_datetime', ALL_PRODUCTS, processing_datetime.strftime(DATE_TIME_METADATA_FORMAT), 'Processing date and time in the format YYYY-MM-DDThh:mm:ss.sZ'], @@ -529,20 +555,28 @@ def get_metadata_dict(product_id: str, # f'site: "Pasadena, CA", ' # f'country: "United States of America"'), # 'Data processing center'], - - # 'identification/CEOSDocumentIdentifier': - # ["https://ceos.org/ard/files/PFS/NRB/v5.5/CARD4L-PFS_NRB_v5.5.pdf", - # 'Product version'], + 'identification/ceosAnalysisReadyDataDocumentIdentifier': + ['ceos_analysis_ready_data_document_identifier', + 'https://ceos.org/ard/files/PFS/NRB/v5.5/CARD4L-PFS_NRB_v5.5.pdf', + ALL_PRODUCTS, + 'CEOS Analysis Ready Data (CARD) document identifier'], + 'identification/dataAccess': + ['product_data_access', + ALL_PRODUCTS, + product_data_access, + 'Location from where this product can be retrieved' + ' (URL or DOI)'], 'metadata/sourceData/numberOfAcquisitions': # 1.6.4 ['source_data_number_of_acquisitions', ALL_PRODUCTS, 1, 'Number of source data acquisitions'], - # TODO Review: should we expose this parameter in the runconfig? - # 'metadata/sourceData/dataAccess': - # ['source_data_access', - # 'https://search.asf.alaska.edu/', - # 'Data access URL'], + 'metadata/sourceData/dataAccess': + ['source_data_access', + ALL_PRODUCTS, + source_data_access, + 'Location from where the source data can be retrieved' + ' (URL or DOI)'], # 'metadata/sourceData/radarBand': # 1.6.4 # ['radar_band', 'C', 'Acquired frequency band'], @@ -570,7 +604,7 @@ def get_metadata_dict(product_id: str, # populate source data processingDateTime with from processing_info # "stop" (SLC Post processing date time) 'metadata/sourceData/processingDateTime': # 1.6.6 - ['source_data_processing_date_time', + ['source_data_processing_datetime', ALL_PRODUCTS, burst_in.burst_misc_metadata.processing_info_dict['stop'], 'Processing UTC date and time of the source data product (SLC' @@ -612,7 +646,7 @@ def get_metadata_dict(product_id: str, ['source_data_zero_doppler_time_spacing', ALL_PRODUCTS, burst_in.azimuth_time_interval, - 'Azimuth spacing of the source data in seconds'], + 'Azimuth spacing of the source data in seconds'], 'metadata/sourceData/slantRangeSpacing': # 1.6.7 ['source_data_slant_range_spacing', ALL_PRODUCTS, @@ -662,44 +696,107 @@ def get_metadata_dict(product_id: str, ALL_PRODUCTS, str(SOFTWARE_VERSION), 'Software version'], - # TODO Review: should we expose this parameter in the runconfig? - # 'metadata/processingInformation/dataAccess': # placeholder for 1.7.1 - # ['product_data_access', - # 'TBD', - # 'URL to access the product data'], - 'metadata/processingInformation/parameters/postProcessingFilteringApplied': # 1.7.4 - ['post_processing_filtering_applied', - STANDARD_RTC_S1_ONLY, + + # 1.7.4 + ('metadata/processingInformation/parameters/' + 'preprocessingMultilookingApplied'): + ['processing_information_multilooking_applied', + ALL_PRODUCTS, + False, + 'Flag to indicate if a preprocessing multilooking has been' + ' applied'], + + # 1.7.4 + ('metadata/processingInformation/parameters/' + 'filteringApplied'): + ['processing_information_filtering_applied', + ALL_PRODUCTS, False, 'Flag to indicate if post-processing filtering has been applied'], # 3.3 'metadata/processingInformation/parameters/noiseCorrectionApplied': - ['noise_correction_applied', + ['processing_information_noise_correction_applied', STANDARD_RTC_S1_ONLY, cfg_in.groups.processing.apply_thermal_noise_correction, 'Flag to indicate if noise removal has been applied'], - 'metadata/processingInformation/parameters/radiometricTerrainCorrectionApplied': - ['rtc_applied', + ('metadata/processingInformation/parameters/' + 'radiometricTerrainCorrectionApplied'): + ['processing_information_radiometric_terrain_correction_applied', STANDARD_RTC_S1_ONLY, cfg_in.groups.processing.apply_rtc, - 'Flag to indicate if radiometric terrain correction (RTC) has been applied'], - 'metadata/processingInformation/parameters/dryTroposphericGeolocationCorrectionApplied': - ['dry_tropospheric_geolocation_correction_applied', - STANDARD_RTC_S1_ONLY, + 'Flag to indicate if radiometric terrain correction (RTC) has' + ' been applied'], + ('metadata/processingInformation/parameters/' + 'dryTroposphericGeolocationCorrectionApplied'): + ['processing_information' + '_dry_tropospheric_geolocation_correction_applied', + ALL_PRODUCTS, cfg_in.groups.processing.apply_dry_tropospheric_delay_correction, - 'Flag to indicate if the dry tropospheric correction has been applied'], - 'metadata/processingInformation/parameters/wetTroposphericGeolocationCorrectionApplied': - ['wet_tropospheric_geolocation_correction_applied', - STANDARD_RTC_S1_ONLY, + 'Flag to indicate if the dry tropospheric correction has been' + ' applied'], + ('metadata/processingInformation/parameters/' + 'wetTroposphericGeolocationCorrectionApplied'): + ['processing_information' + '_wet_tropospheric_geolocation_correction_applied', + ALL_PRODUCTS, False, - 'Flag to indicate if the wet tropospheric correction has been applied'], - 'metadata/processingInformation/parameters/bistaticDelayCorrectionApplied': - ['bistatic_delay_correction_applied', - STANDARD_RTC_S1_ONLY, + 'Flag to indicate if the wet tropospheric correction has been' + ' applied'], + ('metadata/processingInformation/parameters/' + 'bistaticDelayCorrectionApplied'): + ['processing_information' + '_bistatic_delay_correction_applied', + ALL_PRODUCTS, cfg_in.groups.processing.apply_bistatic_delay_correction, - 'Flag to indicate if the bistatic delay correction has been applied'], + 'Flag to indicate if the bistatic delay correction has been' + ' applied'], + ('metadata/processingInformation/parameters/' + 'inputBackscatterNormalizationConvention'): + ['processing_information' + '_input_backscatter_normalization_convention', + ALL_PRODUCTS, + cfg_in.groups.processing.rtc.input_terrain_radiometry, + 'Backscatter normalization convention of the source data'], + ('metadata/processingInformation/parameters/' + 'outputBackscatterNormalizationConvention'): + ['processing_information' + '_output_backscatter_normalization_convention', + ALL_PRODUCTS, + cfg_in.groups.processing.rtc.output_type, + 'Backscatter normalization convention of this product (RTC-S1)'], + + # 3.1 + ('metadata/processingInformation/parameters/' + 'outputBackscatterExpressionConvention'): + ['processing_information' + '_output_backscatter_expression_convention', + ALL_PRODUCTS, + 'linear backscatter intensity', + 'Backscatter expression convension'], + + # 3.2 + ('metadata/processingInformation/parameters/' + 'outputBackscatterDecibelConversionEquation'): + ['processing_information' + '_output_backscatter_decibel_conversion_equation', + ALL_PRODUCTS, + 'backscatter_dB = 10*log10(backscatter_linear)', + 'Equation to convert provided backscatter to decibel (dB)'], + # 4.4 + ('metadata/processingInformation/parameters/geocoding/' + 'burstGeogridSnapX'): + ['processing_information_burst_geogrid_snap_x', + ALL_PRODUCTS, + burst_snap_x, + 'Burst geogrid snap for Coordinate X (W/E)'], + ('metadata/processingInformation/parameters/geocoding/' + 'burstGeogridSnapY'): + ['processing_information_burst_geogrid_snap_y', + ALL_PRODUCTS, + burst_snap_y, + 'Burst geogrid snap for Coordinate Y (S/N)'], # 'metadata/processingInformation/geoidReference': # for 4.2 # 'data/processingInformation/absoluteAccuracyNorthing': @@ -731,17 +828,21 @@ def get_metadata_dict(product_id: str, # applied'], 'metadata/processingInformation/algorithms/demInterpolation': - ['dem_interpolation_algorithm', + ['processing_information' + '_dem_interpolation_algorithm', ALL_PRODUCTS, cfg_in.groups.processing.dem_interpolation_method, 'DEM interpolation method'], 'metadata/processingInformation/algorithms/geocoding': - ['geocoding_algorithm', + ['processing_information' + '_geocoding_algorithm', ALL_PRODUCTS, cfg_in.groups.processing.geocoding.algorithm_type, 'Geocoding algorithm'], - 'metadata/processingInformation/algorithms/radiometricTerrainCorrection': - ['radiometric_terrain_correction_algorithm', + 'metadata/processingInformation/algorithms/' + + 'radiometricTerrainCorrection': + ['processing_information' + '_radiometric_terrain_correction_algorithm', ALL_PRODUCTS, cfg_in.groups.processing.rtc.algorithm_type, 'Radiometric terrain correction (RTC) algorithm'], @@ -759,29 +860,29 @@ def get_metadata_dict(product_id: str, release_version, 'Version of the OPERA s1-reader used for processing'], - 'metadata/processingInformation/inputs/l1SLCGranules': - ['l1_slc_granules', + 'metadata/processingInformation/inputs/l1SlcGranules': + ['inputs_l1_slc_granules', ALL_PRODUCTS, l1_slc_granules, 'List of input L1 SLC products used'], 'metadata/processingInformation/inputs/orbitFiles': - ['orbit_files', + ['inputs_orbit_files', ALL_PRODUCTS, orbit_files, 'List of input orbit files used'], 'metadata/processingInformation/inputs/annotationFiles': - ['annotation_files', + ['inputs_annotation_files', ALL_PRODUCTS, [burst_in.burst_calibration.basename_cads, burst_in.burst_noise.basename_nads], 'List of input annotation files used'], 'metadata/processingInformation/inputs/configFiles': - ['config_files', + ['inputs_config_files', ALL_PRODUCTS, cfg_in.run_config_path, 'List of input config files used'], 'metadata/processingInformation/inputs/demSource': - ['dem_source', + ['inputs_dem_source', ALL_PRODUCTS, dem_file_description, 'Description of the input digital elevation model (DEM)'] @@ -799,7 +900,8 @@ def get_metadata_dict(product_id: str, noise_removal_algorithm_reference = '(noise removal not applied)' metadata_dict['metadata/processingInformation/algorithms/' 'noiseCorrectionAlgorithmReference'] =\ - ['noise_removal_algorithm_reference', + ['processing_information' + '_noise_removal_algorithm_reference', STANDARD_RTC_S1_ONLY, noise_removal_algorithm_reference, 'A reference to the noise removal algorithm applied'] @@ -825,7 +927,8 @@ def get_metadata_dict(product_id: str, raise NotImplementedError metadata_dict['metadata/processingInformation/algorithms/' 'radiometricTerrainCorrectionAlgorithmReference'] =\ - ['rtc_algorithm_reference', + ['processing_information' + '_radiometric_terrain_correction_algorithm_reference', ALL_PRODUCTS, url_rtc_algorithm_document, 'Reference to the radiometric terrain correction (RTC) algorithm' @@ -842,14 +945,29 @@ def get_metadata_dict(product_id: str, ' Art no. 5222723, doi: 10.1109/TGRS.2022.3147472.') metadata_dict['metadata/processingInformation/algorithms/' 'geocodingAlgorithmReference'] =\ - ['geocoding_algorithm_reference', + ['processing_information' + '_geocoding_algorithm_reference', ALL_PRODUCTS, url_geocoding_algorithm_document, 'Reference to the geocoding algorithm applied'] - if not is_mosaic: + if is_mosaic: + # Metadata only for the mosaic product + metadata_dict['metadata/processingInformation/parameters/geocoding/' + 'mosaicGeogridSnapX'] = \ + ['processing_information_mosaic_geogrid_snap_x', + ALL_PRODUCTS, + mosaic_snap_x, + 'mosaic geogrid snap for Coordinate X (W/E)'] + metadata_dict['metadata/processingInformation/parameters/geocoding/' + 'mosaicGeogridSnapY'] = \ + ['processing_information_mosaic_geogrid_snap_y', + ALL_PRODUCTS, + mosaic_snap_y, + 'mosaic geogrid snap for Coordinate Y (S/N)'] - # Metadata only for for burst product + else: + # Metadata only for the burst product # Calculate bounding box xmin_geogrid = cfg_in.geogrids[str(burst_in.burst_id)].start_x ymax_geogrid = cfg_in.geogrids[str(burst_in.burst_id)].start_y @@ -882,14 +1000,14 @@ def get_metadata_dict(product_id: str, metadata_dict['identification/boundingBox'] = \ [None, - STANDARD_RTC_S1_ONLY, + ALL_PRODUCTS, np.array(xy_bounding_box), # 1.7.5 'Bounding box of the product, in order of xmin, ymin, xmax, ymax'] # Attribute `epsg` for HDF5 dataset /identification/boundingBox metadata_dict['identification/boundingBox[epsg]'] = \ [None, - STANDARD_RTC_S1_ONLY, + ALL_PRODUCTS, str(epsg_geogrid), 'Bounding box EPSG code'] @@ -935,19 +1053,19 @@ def get_metadata_dict(product_id: str, ' YYYY-MM-DDThh:mm:ss.sZ'] # 1.6.3 metadata_dict['metadata/sourceData/numberOfAzimuthLines'] = \ ['source_data_number_of_azimuth_lines', - ALL_PRODUCTS, + STANDARD_RTC_S1_ONLY, burst_in.length, 'Number of azimuth lines within the source data product'] metadata_dict['metadata/sourceData/numberOfRangeSamples'] = \ ['source_data_number_of_range_samples', - ALL_PRODUCTS, + STANDARD_RTC_S1_ONLY, burst_in.width, 'Number of slant range samples for each azimuth line within the' ' source data'] metadata_dict['metadata/sourceData/slantRangeStart'] = \ ['source_data_slant_range_start', - ALL_PRODUCTS, + STANDARD_RTC_S1_ONLY, burst_in.starting_range, 'Source data slant range start distance'] @@ -960,7 +1078,7 @@ def get_metadata_dict(product_id: str, this_product_metadata_dict[h5_path] = [geotiff_field, data, description] - if not is_mosaic: + if not is_mosaic and product_type != STATIC_LAYERS_PRODUCT_TYPE: # Add RFI metadata into `metadata_dict` rfi_metadata_dict = get_rfi_metadata_dict(burst_in, @@ -1223,7 +1341,7 @@ def get_rfi_metadata_dict(burst_in, 'Azimuth time of the burst that corresponds to the RFI report' ' in the format YYYY-MM-DDThh:mm:ss.sZ'], 'rfiBurstReport/inBandOutBandPowerRatio': - ['in_band_out_band_power_ratio', + ['rfi_in_band_out_band_power_ratio', burst_in.burst_rfi_info.rfi_burst_report[ 'inBandOutBandPowerRatio'], 'Ratio between the in-band and out-of-band power of the burst.'] diff --git a/src/rtc/mosaic_geobursts.py b/src/rtc/mosaic_geobursts.py index 6dbbd38c..7e87ce20 100644 --- a/src/rtc/mosaic_geobursts.py +++ b/src/rtc/mosaic_geobursts.py @@ -275,18 +275,17 @@ def compute_mosaic_array(list_rtc_images, list_nlooks, mosaic_mode, scratch_dir= wkt_projection = srs_mosaic.ExportToWkt() if verbose: - print(f' mosaic geogrid:') - print(f' start X:', xmin_mosaic) - print(f' end X:', xmax_mosaic) - print(f' start Y:', ymax_mosaic) - print(f' end Y:', ymin_mosaic) - print(f' spacing X:', posting_x) - print(f' spacing Y:', posting_y) - print(f' width:', dim_mosaic[1]) - print(f' length:', dim_mosaic[0]) - print(f' projection:', wkt_projection) - print(f' number of bands: {num_bands}') - + print(' mosaic geogrid:') + print(' start X:', xmin_mosaic) + print(' end X:', xmax_mosaic) + print(' start Y:', ymax_mosaic) + print(' end Y:', ymin_mosaic) + print(' spacing X:', posting_x) + print(' spacing Y:', posting_y) + print(' width:', dim_mosaic[1]) + print(' length:', dim_mosaic[0]) + print(' projection:', wkt_projection) + print(' number of bands:', num_bands) for i, path_rtc in enumerate(list_rtc_images): if i < len(list_nlooks): @@ -516,7 +515,7 @@ def compute_mosaic_array(list_rtc_images, list_nlooks, mosaic_mode, scratch_dir= offset_imgx: offset_imgx + width] = arr_temp rtc_image_gdal_ds = None - + if mosaic_mode.lower() == 'average': # Mode: average # `arr_numerator` holds the accumulated sum. Now, we divide it @@ -616,6 +615,7 @@ def mosaic_single_output_file(list_rtc_images, list_nlooks, mosaic_filename, band_ds = None reference_raster = None + def mosaic_multiple_output_files( list_rtc_images, list_nlooks, output_file_list, mosaic_mode, scratch_dir='', geogrid_in=None, temp_files_list=None, @@ -651,7 +651,7 @@ def mosaic_multiple_output_files( mosaic_dict = compute_mosaic_array( list_rtc_images, list_nlooks, mosaic_mode, scratch_dir=scratch_dir, geogrid_in=geogrid_in, temp_files_list=temp_files_list, - verbose = verbose) + verbose=verbose) arr_numerator = mosaic_dict['mosaic_array'] length = mosaic_dict['length'] diff --git a/src/rtc/rtc_s1.py b/src/rtc/rtc_s1.py index ac5a86a3..320e88f9 100755 --- a/src/rtc/rtc_s1.py +++ b/src/rtc/rtc_s1.py @@ -30,6 +30,12 @@ all_metadata_dict_to_geotiff_metadata_dict, layer_hdf5_dict, DATA_BASE_GROUP, + LAYER_NAME_INCIDENCE_ANGLE, + LAYER_NAME_LOCAL_INCIDENCE_ANGLE, + LAYER_NAME_PROJECTION_ANGLE, + LAYER_NAME_RTC_ANF_PROJECTION_ANGLE, + LAYER_NAME_RANGE_SLOPE, + LAYER_NAME_DEM, LAYER_NAME_LAYOVER_SHADOW_MASK, LAYER_NAME_RTC_ANF_GAMMA0_TO_SIGMA0, LAYER_NAME_NUMBER_OF_LOOKS) @@ -157,17 +163,6 @@ def split_runconfig(cfg_in, 'save_bursts'], True) - # TODO: Remove the code below once the - # mosaicking algorithm does not take nlooks as the weight input - if cfg_in.groups.product_group.save_mosaics: - set_dict_item_recursive(runconfig_dict_out, - ['runconfig', - 'groups', - 'processing', - 'geocoding', - 'save_nlooks'], - True) - runconfig_burst_list.append(path_temp_runconfig) logfile_burst_list.append(path_logfile_child) @@ -271,6 +266,8 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): # primary executable product_type = cfg.groups.primary_executable.product_type product_version_float = cfg.groups.product_group.product_version + rtc_s1_static_validity_start_date = \ + cfg.groups.product_group.rtc_s1_static_validity_start_date if product_version_float is None: product_version = SOFTWARE_VERSION else: @@ -303,7 +300,8 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): 2) mosaic_product_id = populate_product_id( runconfig_product_id, burst_ref, processing_datetime, product_version, - pixel_spacing_avg, is_mosaic=True) + rtc_s1_static_validity_start_date, pixel_spacing_avg, product_type, + is_mosaic=True) # set scratch directory and output_dir scratch_path = os.path.join( @@ -535,7 +533,8 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): pixel_spacing_avg = int((geogrid.spacing_x + geogrid.spacing_y) / 2) burst_product_id = populate_product_id( runconfig_product_id, burst, processing_datetime, product_version, - pixel_spacing_avg, is_mosaic=True) + pixel_spacing_avg, product_type, rtc_s1_static_validity_start_date, + is_mosaic=True) burst_product_id_list.append(burst_product_id) # burst files are saved in scratch dir @@ -758,37 +757,38 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): else: layover_shadow_mask_file = None - # Output imagery list contains multi-band files that - # will be used for mosaicking - output_burst_imagery_list = [] - for pol in pol_list: - if save_imagery_as_hdf5: - geo_burst_pol_filename = (f'NETCDF:{burst_hdf5_in_output}:' - f'{DATA_BASE_GROUP}/' - f'{pol}') + if product_type != STATIC_LAYERS_PRODUCT_TYPE: + # Output imagery list contains multi-band files that + # will be used for mosaicking + output_burst_imagery_list = [] + for pol in pol_list: + if save_imagery_as_hdf5: + geo_burst_pol_filename = (f'NETCDF:{burst_hdf5_in_output}:' + f'{DATA_BASE_GROUP}/' + f'{pol}') + else: + geo_burst_pol_filename = \ + os.path.join(output_path_child, burst_id, + f'{burst_product_id}_{pol}.' + + f'{imagery_extension}') + output_burst_imagery_list.append(geo_burst_pol_filename) + + # Bundle the single-pol geo burst files into .vrt + geo_burst_vrt_filename = geo_burst_filename.replace( + f'.{imagery_extension}', '.vrt') + os.makedirs(os.path.dirname(geo_burst_vrt_filename), exist_ok=True) + gdal.BuildVRT(geo_burst_vrt_filename, output_burst_imagery_list, + options=vrt_options_mosaic) + output_imagery_list.append(geo_burst_vrt_filename) + + # .vrt files (for RTC product in geogrid) will be removed after the + # process + temp_files_list.append(geo_burst_vrt_filename) + + if not flag_bursts_files_are_temporary: + output_file_list += output_burst_imagery_list else: - geo_burst_pol_filename = \ - os.path.join(output_path_child, burst_id, - f'{burst_product_id}_{pol}.' + - f'{imagery_extension}') - output_burst_imagery_list.append(geo_burst_pol_filename) - - # Bundle the single-pol geo burst files into .vrt - geo_burst_vrt_filename = geo_burst_filename.replace( - f'.{imagery_extension}', '.vrt') - os.makedirs(os.path.dirname(geo_burst_vrt_filename), exist_ok=True) - gdal.BuildVRT(geo_burst_vrt_filename, output_burst_imagery_list, - options=vrt_options_mosaic) - output_imagery_list.append(geo_burst_vrt_filename) - - # .vrt files (for RTC product in geogrid) will be removed after the - # process - temp_files_list.append(geo_burst_vrt_filename) - - if not flag_bursts_files_are_temporary: - output_file_list += output_burst_imagery_list - else: - temp_files_list += output_burst_imagery_list + temp_files_list += output_burst_imagery_list if save_nlooks: output_metadata_dict[ @@ -807,12 +807,12 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): # radar-grid layers if flag_call_radar_grid: radar_grid_layer_dict = { - 'incidence_angle': save_incidence_angle, - 'local_incidence_angle': save_local_inc_angle, - 'projection_angle': save_projection_angle, - 'rtc_anf_projection_angle': save_rtc_anf_projection_angle, - 'range_slope': save_range_slope, - 'interpolated_dem': save_dem} + LAYER_NAME_INCIDENCE_ANGLE: save_incidence_angle, + LAYER_NAME_LOCAL_INCIDENCE_ANGLE: save_local_inc_angle, + LAYER_NAME_PROJECTION_ANGLE: save_projection_angle, + LAYER_NAME_RTC_ANF_PROJECTION_ANGLE: save_rtc_anf_projection_angle, + LAYER_NAME_RANGE_SLOPE: save_range_slope, + LAYER_NAME_DEM: save_dem} for layer_name, flag_save in radar_grid_layer_dict.items(): if not flag_save: @@ -882,15 +882,16 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if save_mosaics: - # Mosaic sub-bursts imagery - logger.info('mosaicking files:') - output_imagery_filename_list = [] - for pol in pol_list: - geo_pol_filename = \ - (f'{output_dir_mosaic_raster}/{mosaic_product_id}_{pol}.' - f'{imagery_extension}') - logger.info(f' {geo_pol_filename}') - output_imagery_filename_list.append(geo_pol_filename) + if len(output_imagery_list) > 0: + # Mosaic sub-bursts imagery + logger.info('mosaicking files:') + output_imagery_filename_list = [] + for pol in pol_list: + geo_pol_filename = \ + (f'{output_dir_mosaic_raster}/{mosaic_product_id}_{pol}.' + f'{imagery_extension}') + logger.info(f' {geo_pol_filename}') + output_imagery_filename_list.append(geo_pol_filename) if save_nlooks: nlooks_list = output_metadata_dict[LAYER_NAME_NUMBER_OF_LOOKS][1] @@ -906,13 +907,13 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): temp_files_list=temp_files_list, output_raster_format=output_raster_format) - if save_imagery_as_hdf5: - temp_files_list += output_imagery_filename_list - else: - output_file_list += output_imagery_filename_list - mosaic_output_file_list += output_imagery_filename_list + if save_imagery_as_hdf5: + temp_files_list += output_imagery_filename_list + else: + output_file_list += output_imagery_filename_list + mosaic_output_file_list += output_imagery_filename_list - # Mosaic other bands + # Mosaic other layers for key, (output_file, input_files) in output_metadata_dict.items(): logger.info(f'mosaicking file: {output_file}') if len(input_files) == 0: @@ -924,9 +925,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): temp_files_list=temp_files_list, output_raster_format=output_raster_format) - # TODO: Remove nlooks exception below - if (save_secondary_layers_as_hdf5 or - (key == LAYER_NAME_NUMBER_OF_LOOKS and not save_nlooks)): + if (save_secondary_layers_as_hdf5): temp_files_list.append(output_file) else: output_file_list.append(output_file) @@ -1009,8 +1008,7 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): hdf5_mosaic_obj.close() output_file_list.append(output_hdf5_file) - # Append metadata to mosaic GeoTIFFs - if save_mosaics: + # Append metadata to mosaic GeoTIFFs for current_file in mosaic_output_file_list: if not current_file.endswith('.tif'): continue diff --git a/src/rtc/rtc_s1_single_job.py b/src/rtc/rtc_s1_single_job.py index 010ce058..fdfeed48 100755 --- a/src/rtc/rtc_s1_single_job.py +++ b/src/rtc/rtc_s1_single_job.py @@ -46,10 +46,13 @@ STATIC_LAYERS_LAYOVER_SHADOW_MASK_MULTILOOK_FACTOR = 3 +STATIC_LAYERS_AZ_MARGIN = 1.2 +STATIC_LAYERS_RG_MARGIN = 0.2 def populate_product_id(product_id, burst_in, processing_datetime, - product_version, pixel_spacing, is_mosaic): + product_version, pixel_spacing, product_type, + rtc_s1_static_validity_start_date, is_mosaic): ''' Populate product_id string with S1/RTC-S1 parameters @@ -65,10 +68,16 @@ def populate_product_id(product_id, burst_in, processing_datetime, Product version pixel_spacing: scalar Pixel spacing + product_type: string + Product type + rtc_s1_static_validity_start_date: int + Validity start date (only applicable for the RTC-S1-STATIC product) + in the format YYYYMMDD is_mosaic: bool Flag indicating whether the product ID refers to a mosaic or a burst product + Returns ------- _: str @@ -78,9 +87,19 @@ def populate_product_id(product_id, burst_in, processing_datetime, if product_id is None: product_id = '{product_id}' + if ('{product_id}' in product_id and + product_type != STATIC_LAYERS_PRODUCT_TYPE): + product_id = ('OPERA_L2_RTC-S1_{burst_id}_{sensing_start_datetime}' + '_{processing_datetime}_{sensor}_{pixel_spacing}' + '_{product_version}') if '{product_id}' in product_id: - product_id = ('OPERA_L2_RTC-S1_{burst_id}_{sensing_start_datetime}_' - '{processing_datetime}_{sensor}_{pixel_spacing}' + if not rtc_s1_static_validity_start_date: + error_msg = ('ERROR please provide a' + + ' `rtc_s1_static_validity_start_date`') + raise ValueError(error_msg) + product_id = ('OPERA_L2_RTC-S1-STATIC_{burst_id}' + f'_{rtc_s1_static_validity_start_date}' + '_{processing_datetime}_{sensor}_{pixel_spacing}' '_{product_version}') # Populate product_id sensing_start_datetime @@ -401,6 +420,16 @@ def append_metadata_to_geotiff_file(input_file, metadata_dict, product_id): # Write metadata gdal_ds.SetMetadata(existing_metadata) + # Check NoDataValue + for band in range(gdal_ds.RasterCount): + band_ds = gdal_ds.GetRasterBand(band + 1) + dtype = band_ds.DataType + dtype_name = gdal.GetDataTypeName(dtype) + if ('float' in dtype_name.lower() and + band_ds.GetNoDataValue() is None): + band_ds.SetNoDataValue(np.nan) + del band_ds + # Close GDAL dataset del gdal_ds @@ -750,35 +779,12 @@ def compute_layover_shadow_mask(radar_grid: isce3.product.RadarGridParameters, slantrange_layover_shadow_mask_raster = isce3.io.Raster( path_layover_shadow_mask, radar_grid.width, radar_grid.length, 1, gdal.GDT_Byte, 'MEM') - - # TODO Remove next lines after topo is fixed and X, Y, and inc will - # not be required to compute the layover/shadow mask - x_raster_path = (f'x_{burst_in.burst_id}_' - f'{burst_in.polarization}_{str_datetime}') - x_raster = isce3.io.Raster(x_raster_path, radar_grid.width, - radar_grid.length, - 1, gdal.GDT_Byte, 'MEM') - y_raster_path = (f'x_{burst_in.burst_id}_' - f'{burst_in.polarization}_{str_datetime}') - y_raster = isce3.io.Raster(y_raster_path, radar_grid.width, - radar_grid.length, - 1, gdal.GDT_Byte, 'MEM') - incidence_angle_raster_path = ( - f'x_{burst_in.burst_id}_' - f'{burst_in.polarization}_{str_datetime}') - incidence_angle_raster = isce3.io.Raster(incidence_angle_raster_path, - radar_grid.width, - radar_grid.length, - 1, gdal.GDT_Byte, 'MEM') rdr2geo_obj.topo( dem_raster, - x_raster=x_raster, - y_raster=y_raster, - incidence_angle_raster=incidence_angle_raster, layover_shadow_raster=slantrange_layover_shadow_mask_raster) - if shadow_dilation_size > 0: + if shadow_dilation_size > 1: ''' constants from ISCE3: SHADOW_VALUE = 1; @@ -938,6 +944,8 @@ def run_single_job(cfg: RunConfig): # primary executable product_type = cfg.groups.primary_executable.product_type product_version_float = cfg.groups.product_group.product_version + rtc_s1_static_validity_start_date = \ + cfg.groups.product_group.rtc_s1_static_validity_start_date if product_version_float is None: product_version = SOFTWARE_VERSION else: @@ -970,7 +978,7 @@ def run_single_job(cfg: RunConfig): 2) mosaic_product_id = populate_product_id( runconfig_product_id, burst_ref, processing_datetime, product_version, - pixel_spacing_avg, is_mosaic=True) + pixel_spacing_avg, product_type, rtc_s1_static_validity_start_date, is_mosaic=True) scratch_path = os.path.join( cfg.groups.product_group.scratch_path, f'temp_{time_stamp}') @@ -1057,7 +1065,8 @@ def run_single_job(cfg: RunConfig): save_incidence_angle = geocode_namespace.save_incidence_angle save_local_inc_angle = geocode_namespace.save_local_inc_angle save_projection_angle = geocode_namespace.save_projection_angle - save_rtc_anf_projection_angle = geocode_namespace.save_rtc_anf_projection_angle + save_rtc_anf_projection_angle = \ + geocode_namespace.save_rtc_anf_projection_angle save_range_slope = geocode_namespace.save_range_slope save_nlooks = geocode_namespace.save_nlooks @@ -1102,6 +1111,21 @@ def run_single_job(cfg: RunConfig): rtc_upsampling = rtc_namespace.dem_upsampling rtc_area_beta_mode = rtc_namespace.area_beta_mode + if rtc_area_beta_mode == 'pixel_area': + rtc_area_beta_mode_enum = \ + isce3.geometry.RtcAreaBetaMode.PIXEL_AREA + elif rtc_area_beta_mode == 'projection_angle': + rtc_area_beta_mode_enum = \ + isce3.geometry.RtcAreaBetaMode.PROJECTION_ANGLE + elif (rtc_area_beta_mode == 'auto' or + rtc_area_beta_mode is None): + rtc_area_beta_mode_enum = \ + isce3.geometry.RtcAreaBetaMode.AUTO + else: + err_msg = ('ERROR invalid area beta mode:' + f' {rtc_area_beta_mode}') + raise ValueError(err_msg) + logger.info('Identification:') logger.info(f' product type: {product_type}') logger.info(f' product version: {product_version}') @@ -1251,12 +1275,13 @@ def run_single_job(cfg: RunConfig): geogrid = cfg.geogrids[burst_id] pol_list = list(burst_pol_dict.keys()) burst = burst_pol_dict[pol_list[0]] - + # populate burst_product_id pixel_spacing_avg = int((geogrid.spacing_x + geogrid.spacing_y) / 2) burst_product_id = populate_product_id( runconfig_product_id, burst, processing_datetime, product_version, - pixel_spacing_avg, is_mosaic=True) + pixel_spacing_avg, product_type, rtc_s1_static_validity_start_date, + is_mosaic=True) logger.info(f' product ID: {burst_product_id}') @@ -1334,10 +1359,10 @@ def run_single_job(cfg: RunConfig): radar_grid = burst.as_isce3_radargrid() if product_type == STATIC_LAYERS_PRODUCT_TYPE: radar_grid = radar_grid.offset_and_resize( - - int(1.5 * radar_grid.length), - - int(radar_grid.width), - int(4 * radar_grid.length), - int(3 * radar_grid.width)) + - int((STATIC_LAYERS_AZ_MARGIN) * radar_grid.length), + - int((STATIC_LAYERS_RG_MARGIN) * radar_grid.width), + int((1 + 2 * STATIC_LAYERS_AZ_MARGIN) * radar_grid.length), + int((1 + 2 * STATIC_LAYERS_RG_MARGIN) * radar_grid.width)) # native_doppler = burst.doppler.lut2d orbit = burst.orbit wavelength = burst.wavelength @@ -1600,35 +1625,6 @@ def run_single_job(cfg: RunConfig): geogrid.spacing_x, geogrid.spacing_y, geogrid.width, geogrid.length, geogrid.epsg) - if rtc_area_beta_mode != 'auto': - - - - - - # TODO! This code should be moved to runconfig after `area_beta_mode` - # is added to geocode() in ISCE3 - if rtc_area_beta_mode == 'pixel_area': - rtc_area_beta_mode_enum = \ - isce3.geometry.RtcAreaBetaMode.PIXEL_AREA - elif rtc_area_beta_mode == 'projection_angle': - rtc_area_beta_mode_enum = \ - isce3.geometry.RtcAreaBetaMode.PROJECTION_ANGLE - elif (rtc_area_beta_mode == 'auto' or - rtc_area_beta_mode is None): - rtc_area_beta_mode_enum = \ - isce3.geometry.RtcAreaBetaMode.AUTO - else: - err_msg = ('ERROR invalid area beta mode:' - f' {rtc_area_beta_mode}') - raise ValueError(err_msg) - - - - - - geocode_kwargs['rtc_area_beta_mode'] = rtc_area_beta_mode_enum - geo_obj.geocode(radar_grid=radar_grid, input_raster=rdr_burst_raster, output_raster=geo_burst_raster, @@ -1648,6 +1644,7 @@ def run_single_job(cfg: RunConfig): clip_max=clip_max, out_geo_nlooks=out_geo_nlooks_obj, out_geo_rtc=out_geo_rtc_obj, + rtc_area_beta_mode=rtc_area_beta_mode_enum, # out_geo_rtc_gamma0_to_sigma0=out_geo_rtc_gamma0_to_sigma0_obj, input_rtc=None, output_rtc=None, @@ -1659,7 +1656,8 @@ def run_single_job(cfg: RunConfig): # Output imagery list contains multi-band files that # will be used for mosaicking - output_imagery_list.append(geo_burst_filename) + if product_type != STATIC_LAYERS_PRODUCT_TYPE: + output_imagery_list.append(geo_burst_filename) else: # Bundle the single-pol geo burst files into .vrt @@ -1668,7 +1666,8 @@ def run_single_job(cfg: RunConfig): os.makedirs(os.path.dirname(geo_burst_filename), exist_ok=True) gdal.BuildVRT(geo_burst_filename, output_burst_imagery_list, options=vrt_options_mosaic) - output_imagery_list.append(geo_burst_filename) + if product_type != STATIC_LAYERS_PRODUCT_TYPE: + output_imagery_list.append(geo_burst_filename) if (flag_process and save_layover_shadow_mask and not save_secondary_layers_as_hdf5): @@ -1676,7 +1675,8 @@ def run_single_job(cfg: RunConfig): geo_burst_filename) # If burst imagery is not temporary, separate polarization channels - if flag_process and not flag_bursts_files_are_temporary: + if (flag_process and not flag_bursts_files_are_temporary and + product_type != STATIC_LAYERS_PRODUCT_TYPE): _separate_pol_channels(geo_burst_filename, output_burst_imagery_list, output_raster_format, logger) @@ -1684,29 +1684,32 @@ def run_single_job(cfg: RunConfig): output_file_list += output_burst_imagery_list if save_nlooks: - out_geo_nlooks_obj.close_dataset() - del out_geo_nlooks_obj + if flag_process: + out_geo_nlooks_obj.close_dataset() + del out_geo_nlooks_obj - if not flag_bursts_secondary_files_are_temporary: - logger.info(f'file saved: {nlooks_file}') + if not flag_bursts_secondary_files_are_temporary: + logger.info(f'file saved: {nlooks_file}') output_metadata_dict[ LAYER_NAME_NUMBER_OF_LOOKS][1].append(nlooks_file) if save_rtc_anf: - out_geo_rtc_obj.close_dataset() - del out_geo_rtc_obj + if flag_process: + out_geo_rtc_obj.close_dataset() + del out_geo_rtc_obj - if not flag_bursts_secondary_files_are_temporary: - logger.info(f'file saved: {rtc_anf_file}') + if not flag_bursts_secondary_files_are_temporary: + logger.info(f'file saved: {rtc_anf_file}') output_metadata_dict[layer_name_rtc_anf][1].append( rtc_anf_file) if save_rtc_anf_gamma0_to_sigma0: - out_geo_rtc_gamma0_to_sigma0_obj.close_dataset() - del out_geo_rtc_gamma0_to_sigma0_obj + if flag_process: + out_geo_rtc_gamma0_to_sigma0_obj.close_dataset() + del out_geo_rtc_gamma0_to_sigma0_obj - if not flag_bursts_secondary_files_are_temporary: - logger.info(f'file saved: {rtc_anf_gamma0_to_sigma0_file}') + if not flag_bursts_secondary_files_are_temporary: + logger.info(f'file saved: {rtc_anf_gamma0_to_sigma0_file}') output_metadata_dict[ LAYER_NAME_RTC_ANF_GAMMA0_TO_SIGMA0][1].append( rtc_anf_gamma0_to_sigma0_file) @@ -1821,15 +1824,16 @@ def run_single_job(cfg: RunConfig): if save_mosaics: - # Mosaic sub-bursts imagery - logger.info('mosaicking files:') - output_imagery_filename_list = [] - for pol in pol_list: - geo_pol_filename = \ - (f'{output_dir_mosaic_raster}/{mosaic_product_id}_{pol}.' - f'{imagery_extension}') - logger.info(f' {geo_pol_filename}') - output_imagery_filename_list.append(geo_pol_filename) + if len(output_imagery_list) > 0: + # Mosaic sub-bursts imagery + logger.info('mosaicking files:') + output_imagery_filename_list = [] + for pol in pol_list: + geo_pol_filename = \ + (f'{output_dir_mosaic_raster}/{mosaic_product_id}_{pol}.' + f'{imagery_extension}') + logger.info(f' {geo_pol_filename}') + output_imagery_filename_list.append(geo_pol_filename) if save_nlooks: nlooks_list = output_metadata_dict[LAYER_NAME_NUMBER_OF_LOOKS][1] @@ -1845,13 +1849,13 @@ def run_single_job(cfg: RunConfig): temp_files_list=temp_files_list, output_raster_format=output_raster_format) - if save_imagery_as_hdf5: - temp_files_list += output_imagery_filename_list - else: - output_file_list += output_imagery_filename_list - mosaic_output_file_list += output_imagery_filename_list + if save_imagery_as_hdf5: + temp_files_list += output_imagery_filename_list + else: + output_file_list += output_imagery_filename_list + mosaic_output_file_list += output_imagery_filename_list - # Mosaic other bands + # Mosaic other layers for key, (output_file, input_files) in output_metadata_dict.items(): logger.info(f'mosaicking file: {output_file}') if len(input_files) == 0: @@ -1863,9 +1867,7 @@ def run_single_job(cfg: RunConfig): geogrid_in=cfg.geogrid, temp_files_list=temp_files_list, output_raster_format=output_raster_format) - # TODO: Remove nlooks exception below - if (save_secondary_layers_as_hdf5 or - (key == LAYER_NAME_NUMBER_OF_LOOKS and not save_nlooks)): + if save_secondary_layers_as_hdf5: temp_files_list.append(output_file) else: output_file_list.append(output_file) @@ -1948,8 +1950,7 @@ def run_single_job(cfg: RunConfig): hdf5_mosaic_obj.close() output_file_list.append(output_hdf5_file) - # Append metadata to mosaic GeoTIFFs - if save_mosaics: + # Append metadata to mosaic GeoTIFFs for current_file in mosaic_output_file_list: if not current_file.endswith('.tif'): continue diff --git a/src/rtc/runconfig.py b/src/rtc/runconfig.py index 9198c1e3..c2a6b67e 100644 --- a/src/rtc/runconfig.py +++ b/src/rtc/runconfig.py @@ -20,7 +20,7 @@ from s1reader.s1_orbit import get_orbit_file_from_list from s1reader.s1_reader import load_bursts -STATIC_LAYERS_PRODUCT_TYPE = 'RTC_S1_SL' +STATIC_LAYERS_PRODUCT_TYPE = 'RTC_S1_STATIC' logger = logging.getLogger('rtc_s1') @@ -147,7 +147,7 @@ def load_validate_yaml(yaml_path: str) -> dict: 'product_type'] if (product_type == STATIC_LAYERS_PRODUCT_TYPE): default_cfg_path = (f'{helpers.WORKFLOW_SCRIPTS_DIR}/defaults/' - f'rtc_s1_sl.yaml') + f'rtc_s1_static.yaml') print('Loading RTC-S1 runconfig default for static layers') else: default_cfg_path = (f'{helpers.WORKFLOW_SCRIPTS_DIR}/defaults/' diff --git a/src/rtc/schemas/rtc_s1.yaml b/src/rtc/schemas/rtc_s1.yaml index 7dd1db51..0c734b7e 100644 --- a/src/rtc/schemas/rtc_s1.yaml +++ b/src/rtc/schemas/rtc_s1.yaml @@ -5,8 +5,8 @@ runconfig: primary_executable: - # Required. Output product type: "RTC_S1" or "RTC_S1_SL" - product_type: enum('RTC_S1', 'RTC_S1_SL') + # Required. Output product type: "RTC_S1" or "RTC_S1_STATIC" + product_type: enum('RTC_S1', 'RTC_S1_STATIC') pge_name_group: pge_name: enum('RTC_S1_PGE') @@ -14,8 +14,13 @@ runconfig: input_file_group: # Required. List of SAFE files (min=1) safe_file_path: list(str(), min=1) + + # Location from where the source data can be retrieved (URL or DOI) + source_data_access: str(required=False) + # Required. List of orbit (EOF) files orbit_file_path: list(str(), min=1) + # Optional. Burst ID to process (empty for all bursts) burst_id: list(str(), min=1, required=False) @@ -43,9 +48,9 @@ runconfig: scratch_path: str() # If option `save_bursts` is set, output bursts are saved to: - # {output_dir}/{burst_id}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} # If option `save_mosaics` is set, output mosaics are saved to: - # {output_dir}/{product_id}_v{product_version}{suffix}.{ext} + # {output_dir}/{product_id}{suffix}.{ext} # # If the `product_id` contains the substring "_{burst_id}", the # substring will be substituted by either: @@ -56,17 +61,30 @@ runconfig: # `RTC-S1_069-147170-IW1_S1B` for the burst t069-147170-IW1; and it # will become `RTC-S1_S1B` for the mosaic product. # - # If the field `product_id`` is left empty, the product ID will - # follow the RTC-S1 file naming conventions: + # For RTC-S1 products, if the field `product_id`` is left empty, + # the burst product ID will follow the RTC-S1 file naming conventions: # `OPERA_L2_RTC-S1_{burst_id}_{sensing_start_datetime}_ # {processing_datetime}_{sensor}_{pixel_spacing} # _{product_version}`. # + # For RTC-S1-STATIC products, if the field `product_id` is left empty, + # the burst product ID will follow the RTC-S1-STATIC file naming + # conventions: + # `OPERA_L2_RTC-S1-STATIC_{burst_id}_{rtc_s1_static_validity_start_date}_ + # {processing_datetime}_{sensor}_{pixel_spacing} + # _{product_version}`. + # # `suffix` is only used when there are multiple output files. # `ext` is determined by geocoding_options.output_imagery_format. output_dir: str() product_id: str(required=False) + # Validity start date for RTC-S1-STATIC products in the format YYYYMMDD + rtc_s1_static_validity_start_date: int(min=20000101, max=21991231,required=False) + + # Location from where the output product can be retrieved (URL or DOI) + product_data_access: str(required=False) + # Save RTC-S1 products save_bursts: bool(required=False)