Merge branch 'develop' into feature/GHRSST_sst

parm/atm/obs/testing/gps_bend.yaml

@@ -0,0 +1,86 @@
+obs space:
+  name: gnssrobndnbam
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV gps_bend_obs_${CDATE}.nc4 
+    obsgrouping:
+      group variables: [ 'sequenceNumber' ]
+      sort variable: 'impactHeightRO'
+      sort order: 'ascending'
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV gps_bend_diag_${CDATE}.nc4 
+  simulated variables: [bendingAngle]
+geovals:
+  filename: !ENV gps_bend_geoval_${CDATE}.nc4
+obs operator:
+  name: GnssroBndNBAM
+  obs options:
+    use_compress: 1
+    sr_steps: 2
+    vertlayer: full
+    super_ref_qc: NBAM
+obs filters:
+#1. gpstop
+- filter: Domain Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/impactHeightRO
+    minvalue: 0
+    maxvalue: 55000.1
+  action:
+    name: reject
+#2. commgpstop
+- filter: Bounds Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/satelliteIdentifier
+    is_in: 265,266,267,268,269
+  test variables:
+  - name: MetaData/impactHeightRO
+  maxvalue: 45000.1
+  action:
+    name: reject
+#3. metop below 8 km
+- filter: Bounds Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/satelliteIdentifier
+    is_in: 3-5
+  test variables:
+  - name: MetaData/impactHeightRO
+  minvalue: 8000.1
+  action:
+    name: reject
+#4. assign obs error 
+- filter: ROobserror
+  filter variables:
+  - name: bendingAngle
+  errmodel: NBAM
+#5. RONBAM cut off check
+- filter: Background Check RONBAM
+  filter variables:
+  - name: bendingAngle
+#6. Obs error inflate
+- filter: Background Check RONBAM
+  filter variables:
+  - name: bendingAngle
+  action:
+    name: RONBAMErrInflate
+#7. Background check
+#- filter: Background Check
+#  filter variables:
+#  - name: bendingAngle
+#  threshold: 10
+#  action:
+#    name: reject
+
+passedBenchmark: 393457

utils/test/prepdata.sh

@@ -5,4 +5,4 @@ project_source_dir=$1
 
 ncgen -o rads_adt_j3_2021182.nc4 ${project_source_dir}/testdata/rads_adt_j3_2021182.cdl
 ncgen -o ghrsst_sst_mb_202107010000.cdl ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl
-ncgen -o ghrsst_sst_mb_202107010100.cdl ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl
+ncgen -o ghrsst_sst_mb_202107010100.cdl ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl

utils/test/testdata/icec_amsr2_north_1.cdl

@@ -0,0 +1,208 @@
+netcdf output {
+dimensions:
+	Number_of_X_Dimension = 3 ;
+	Number_of_Y_Dimension = 4 ;
+	QM = 1 ;
+	Time_Dimension = 6 ;
+variables:
+	short Across_X_Dimension(Number_of_X_Dimension) ;
+		Across_X_Dimension:long_name = " Across X Dimension" ;
+		Across_X_Dimension:coordinates = "" ;
+		Across_X_Dimension:units = "1" ;
+		Across_X_Dimension:_FillValue = -9999s ;
+	short Along_Y_Dimension(Number_of_Y_Dimension) ;
+		Along_Y_Dimension:long_name = " Along Y Dimension" ;
+		Along_Y_Dimension:coordinates = "" ;
+		Along_Y_Dimension:units = "1" ;
+		Along_Y_Dimension:_FillValue = -9999s ;
+	int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction) - secondary product" ;
+		Bootstrap_Ice_Concentration:units = "1" ;
+		Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ;
+		Bootstrap_Ice_Concentration:_FillValue = -9999 ;
+	short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Flags:long_name = "Quality Flags" ;
+		Flags:units = "1" ;
+		Flags:coordinate = "Latitude Longitude" ;
+		Flags:_FillValue = -9999s ;
+		Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ;
+	int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Latency:long_name = "Time in seconds from most current time" ;
+		Latency:units = "seconds" ;
+		Latency:coordinate = "Latitude Longitude" ;
+		Latency:_FillValue = -9999 ;
+	float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Latitude:long_name = "Latitude for EASE-2 grid" ;
+		Latitude:units = "degrees_north" ;
+		Latitude:valid_range = -90.f, 90.f ;
+		Latitude:_FillValue = -9999.f ;
+		Latitude:standard_name = "latitude" ;
+	float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Longitude:long_name = "Longitude for EASE-2 grid" ;
+		Longitude:units = "degrees_east" ;
+		Longitude:valid_range = -180.f, 180.f ;
+		Longitude:_FillValue = -9999.f ;
+		Longitude:standard_name = "longitude" ;
+	int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction) - primary product" ;
+		NASA_Team_2_Ice_Concentration:units = "1" ;
+		NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ;
+		NASA_Team_2_Ice_Concentration:_FillValue = -9999 ;
+	int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ;
+		NASA_Team_2_Multiyear_Ice:units = "1" ;
+		NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ;
+		NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ;
+	int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (fraction)" ;
+		NT2_minus_Bootstrap:units = "1" ;
+		NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ;
+		NT2_minus_Bootstrap:_FillValue = -9999 ;
+	float QM_Num_Grid_Range_25(QM) ;
+		QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ;
+		QM_Num_Grid_Range_25:units = "1" ;
+		QM_Num_Grid_Range_25:_FillValue = -9999.f ;
+	float QM_Num_Grid_Range_50(QM) ;
+		QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes    with Range > 50%" ;
+		QM_Num_Grid_Range_50:units = "1" ;
+		QM_Num_Grid_Range_50:_FillValue = -9999.f ;
+	float QM_Total_Ice(QM) ;
+		QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ;
+		QM_Total_Ice:units = "1" ;
+		QM_Total_Ice:_FillValue = -9999.f ;
+	float QM_Total_Pixels(QM) ;
+		QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ;
+		QM_Total_Pixels:units = "1" ;
+		QM_Total_Pixels:_FillValue = -9999.f ;
+	int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ;
+		Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction) - based on NT2" ;
+		Range_of_Ice_Concentration:units = "1" ;
+		Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ;
+		Range_of_Ice_Concentration:_FillValue = -9999 ;
+	float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ;
+		Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM,    SS) in GMT" ;
+		Scan_Time:units = "1" ;
+		Scan_Time:_FillValue = -9999.f ;
+
+// global attributes:
+		:Conventions = "CF-1.5" ;
+		:Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ;
+		:standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ;
+		:project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ;
+		:title = "AMSR2_SEAICE" ;
+		:summary = "GCOM Seaice Products" ;
+		:institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations,     NESDIS, NOAA, U.S. Department of Commerce" ;
+		:naming_authority = "gov.noaa.nesdis.nde" ;
+		:platform_name = "GCOM-W1" ;
+		:instrument_name = "AMSR2" ;
+		:creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite   Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ;
+		:creator_email = "[email protected]" ;
+		:creator_url = "http://www.star.nesdis.noaa.gov" ;
+		:publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ;
+		:publisher_email = "[email protected]" ;
+		:publisher_url = "http://www.ospo.noaa.gov/" ;
+		:references = "Contact the OSPO PAL to request the ATBD." ;
+		:processing_level = "NOAA Level 2 data" ;
+		:cdm_data_type = "Grid" ;
+		:geospatial_lat_min = 18.87699f ;
+		:geospatial_lat_max = 89.9367f ;
+		:geospatial_lon_min = -179.945f ;
+		:geospatial_lon_max = 179.945f ;
+		:geospatial_lat_units = "degrees_north" ;
+		:geospatial_lon_units = "degrees_east" ;
+		:date_created = "2021-07-01T02:02:39Z" ;
+		:id = "abb20bb6-945f-4f02-a7a5-14f6dcdce4cf" ;
+		:Metadata_Link = "AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc" ;
+		:history = "Mon Sep 25 15:10:33 2023: ncks -d Number_of_X_Dimension,400,600,100 -d Number_of_Y_Dimension,400,700,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc output.nc\nCreated by GAASP Version 2.0, Release 2.0" ;
+		:source = "GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5, GAASP-L1R_v2r2_GW1_s202106292259160_e202106300038160_c202106300112300.h5" ;
+		:production_site = "NSOF" ;
+		:production_environment = "OE" ;
+		:start_orbit_number = 48513 ;
+		:end_orbit_number = 48514 ;
+		:day_night_data_flag = 2 ;
+		:ascend_descend_data_flag = 2 ;
+		:time_coverage_start = "2021-06-30T23:41:19.004Z" ;
+		:time_coverage_end = "2021-07-01T01:20:16.940Z" ;
+		:NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ;
+data:
+
+ Across_X_Dimension = 401, 501, 601 ;
+
+ Along_Y_Dimension = 401, 501, 601, 701 ;
+
+ Bootstrap_Ice_Concentration =
+  100, 100, 0,
+  _, 100, 100,
+  _, 87, 96,
+  _, 0, 0 ;
+
+ Flags =
+  0, 0, 8,
+  120, 0, 0,
+  120, 0, 0,
+  120, 8, 8 ;
+
+ Latency =
+  204, 104, 6,
+  2000, 299, 3,
+  2000, 696, 1,
+  2000, 699, 897 ;
+
+ Latitude =
+  74.18227, 78.6195, 76.93348,
+  78.6195, 86.89752, 82.88853,
+  76.93348, 82.88853, 80.42855,
+  70.63603, 74.07984, 72.82585 ;
+
+ Longitude =
+  -135, -168.8672, 148.7663,
+  -101.1328, -135, 107.9784,
+  -58.7663, -17.97842, 45,
+  -35.35196, -7.947196, 23.27735 ;
+
+ NASA_Team_2_Ice_Concentration =
+  100, 100, 0,
+  _, 100, 100,
+  _, 97, 100,
+  _, 0, 0 ;
+
+ NASA_Team_2_Multiyear_Ice =
+  0, 0, 0,
+  _, 100, 100,
+  _, 0, 0,
+  _, 0, 0 ;
+
+ NT2_minus_Bootstrap =
+  0, 0, 0,
+  _, 0, 0,
+  _, 10, 4,
+  _, 0, 0 ;
+
+ QM_Num_Grid_Range_25 = 10331 ;
+
+ QM_Num_Grid_Range_50 = 2560 ;
+
+ QM_Total_Ice = 98806 ;
+
+ QM_Total_Pixels = 1102500 ;
+
+ Range_of_Ice_Concentration =
+  1, 3, 0,
+  _, 0, 3,
+  _, 13, 4,
+  _, 0, 0 ;
+
+ Scan_Time =
+  2021, 6, 30, 21, 55, 22.60762,
+  2021, 6, 30, 23, 35, 23.53783,
+  2021, 7, 1, 1, 13, 57.47626,
+  _, _, _, _, _, _,
+  2021, 6, 30, 20, 20, 29.15068,
+  2021, 7, 1, 1, 16, 25.96225,
+  _, _, _, _, _, _,
+  2021, 6, 30, 13, 43, 40.417,
+  2021, 7, 1, 1, 18, 54.4482,
+  _, _, _, _, _, _,
+  2021, 6, 30, 13, 41, 11.93098,
+  2021, 6, 30, 10, 22, 37.06214 ;
+}