diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h index ea367b145..6ec1c233e 100644 --- a/utils/obsproc/NetCDFToIodaConverter.h +++ b/utils/obsproc/NetCDFToIodaConverter.h @@ -26,6 +26,13 @@ namespace gdasapp { Eigen::ArrayXf obsError; Eigen::ArrayXi preQc; std::string units; // reference date for epoch time + + explicit IodaVars(const int nobs = 0) : location(nobs), + nVars(1), + obsVal(location), + obsError(location), + preQc(location) + {} }; // Base class for the converters @@ -58,50 +65,77 @@ namespace gdasapp { // Method to write out a IODA file (writter called in destructor) void writeToIoda() { - // Create empty group backed by HDF file - ioda::Group group = - ioda::Engines::HH::createFile( - outputFilename_, - ioda::Engines::BackendCreateModes::Truncate_If_Exists); + // Get communicator + const eckit::mpi::Comm & comm = oops::mpi::world(); // Extract ioda variables from the provider's files - std::string fileName = inputFilenames_[0]; // TODO(Guillaume): make it work for a list gdasapp::IodaVars iodaVars; - providerToIodaVars(fileName, iodaVars); - oops::Log::debug() << "--- iodaVars.location: " << iodaVars.location << std::endl; - oops::Log::debug() << "--- iodaVars.obsVal: " << iodaVars.obsVal << std::endl; - - // Update the group with the location dimension - ioda::NewDimensionScales_t - newDims {ioda::NewDimensionScale("Location", iodaVars.location)}; - ioda::ObsGroup ogrp = ioda::ObsGroup::generate(group, newDims); - - // Set up the float creation parameters - ioda::VariableCreationParameters float_params; - float_params.chunk = true; // allow chunking - float_params.compressWithGZIP(); // compress using gzip - float missing_value = util::missingValue(missing_value); - float_params.setFillValue(missing_value); - - // Create the IODA variables - ioda::Variable adtIodaDatetime = - ogrp.vars.createWithScales("Metadata/dateTime", - {ogrp.vars["Location"]}, float_params); - // TODO(Mindo): Get the date info from the netcdf file - adtIodaDatetime.atts.add("units", {"seconds since 9999-04-15T12:00:00Z"}, {1}); - - ioda::Variable adtIodaObsVal = - ogrp.vars.createWithScales("ObsValue/"+variable_, - {ogrp.vars["Location"]}, float_params); - ioda::Variable adtIodaObsErr = - ogrp.vars.createWithScales("ObsError/"+variable_, - {ogrp.vars["Location"]}, float_params); - - // Write adt obs value to group - adtIodaObsVal.writeWithEigenRegular(iodaVars.obsVal); - - // Write adt obs error to group - adtIodaObsErr.writeWithEigenRegular(iodaVars.obsError); + int myrank = comm.rank(); + int nobs(0); + oops::Log::debug() << "ooooooooooooo my rank : " << myrank << comm.size() << std::endl; + if (myrank <= inputFilenames_.size() - 1) { + providerToIodaVars(inputFilenames_[myrank], iodaVars); + nobs = iodaVars.location; + oops::Log::debug() << "--- iodaVars.location: " << iodaVars.location << std::endl; + oops::Log::debug() << "--- iodaVars.obsVal: " << iodaVars.obsVal << std::endl; + } + + // Get the total number of obs across pe's + comm.allReduce(nobs, nobs, eckit::mpi::sum()); + oops::Log::debug() << " my rank : " << myrank + << " Num pe's: " << comm.size() + << " nobs: " << nobs << std::endl; + gdasapp::IodaVars iodaVarsAll(nobs); + + // Gather obsVal's + gatherObs(comm, iodaVars.obsVal, iodaVarsAll.obsVal); + gatherObs(comm, iodaVars.obsError, iodaVarsAll.obsError); + gatherObs(comm, iodaVars.preQc, iodaVarsAll.preQc); + + oops::Log::debug() << "--- all nobs: " << iodaVarsAll.obsVal.size() << std::endl; + oops::Log::debug() << "--- all obsVal: " << iodaVarsAll.obsVal << std::endl; + oops::Log::debug() << "--- all obsError: " << iodaVarsAll.obsError << std::endl; + oops::Log::debug() << "--- all preQc: " << iodaVarsAll.preQc << std::endl; + + // Create empty group backed by HDF file + if (oops::mpi::world().rank() == 0) { + ioda::Group group = + ioda::Engines::HH::createFile( + outputFilename_, + ioda::Engines::BackendCreateModes::Truncate_If_Exists); + + // Update the group with the location dimension + ioda::NewDimensionScales_t + newDims {ioda::NewDimensionScale("Location", iodaVarsAll.location)}; + ioda::ObsGroup ogrp = ioda::ObsGroup::generate(group, newDims); + + // Set up the creation parameters + ioda::VariableCreationParameters float_params = createVariableParams(); + ioda::VariableCreationParameters int_params = createVariableParams(); + + // Create the IODA variables + ioda::Variable adtIodaDatetime = + ogrp.vars.createWithScales("Metadata/dateTime", + {ogrp.vars["Location"]}, float_params); + // TODO(All): Decide on what to use for the Epoch date + adtIodaDatetime.atts.add("units", {"seconds since 9999-04-15T12:00:00Z"}, {1}); + + ioda::Variable adtIodaObsVal = + ogrp.vars.createWithScales("ObsValue/"+variable_, + {ogrp.vars["Location"]}, float_params); + ioda::Variable adtIodaObsErr = + ogrp.vars.createWithScales("ObsError/"+variable_, + {ogrp.vars["Location"]}, float_params); + + ioda::Variable adtIodaPreQc = + ogrp.vars.createWithScales("PreQC/"+variable_, + {ogrp.vars["Location"]}, int_params); + + // Write adt obs info to group + adtIodaObsVal.writeWithEigenRegular(iodaVarsAll.obsVal); + adtIodaObsErr.writeWithEigenRegular(iodaVarsAll.obsError); + adtIodaPreQc.writeWithEigenRegular(iodaVarsAll.preQc); + } } private: @@ -109,6 +143,29 @@ namespace gdasapp { // info in a IodaVars struct virtual void providerToIodaVars(const std::string fileName, gdasapp::IodaVars & iodaVars) = 0; + // Gather for eigen array + template + void gatherObs(const eckit::mpi::Comm & comm, + const Eigen::Array & obsPe, + Eigen::Array & obsAllPes) { + std::vector tmpVec(obsPe.data(), obsPe.data() + obsPe.size()); + oops::mpi::allGatherv(comm, tmpVec); + for (int i = 0; i < tmpVec.size(); ++i) { + obsAllPes(i) = tmpVec[i]; + } + } + + // Short-cut to create type dependent VariableCreationParameters + template + ioda::VariableCreationParameters createVariableParams() { + ioda::VariableCreationParameters params; + params.chunk = true; // allow chunking + params.compressWithGZIP(); // compress using gzip + params.setFillValue(util::missingValue()); + + return params; + } + protected: util::DateTime windowBegin_; util::DateTime windowEnd_; diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh index 84895d5ef..013c72f38 100755 --- a/utils/test/prepdata.sh +++ b/utils/test/prepdata.sh @@ -9,3 +9,5 @@ ncgen -o icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_ ncgen -o icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl ncgen -o icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl ncgen -o icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl +ncgen -o ghrsst_sst_mb_202107010000.cdl ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl +ncgen -o ghrsst_sst_mb_202107010100.cdl ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl diff --git a/utils/test/testdata/ghrsst_sst_mb_202107010000.cdl b/utils/test/testdata/ghrsst_sst_mb_202107010000.cdl new file mode 100644 index 000000000..a75123873 --- /dev/null +++ b/utils/test/testdata/ghrsst_sst_mb_202107010000.cdl @@ -0,0 +1,295 @@ +netcdf sst_MB_202107010000 { +dimensions: + time = 1 ; + lat = 6 ; + lon = 6 ; +variables: + int crs ; + crs:grid_mapping_name = "latitude_longitude" ; + crs:horizontal_datum_name = "World Geodetic System 1984" ; + byte dt_analysis(time, lat, lon) ; + dt_analysis:add_offset = 0.f ; + dt_analysis:comment = "Deviation from reference SST, i.e., dt_analysis = SST - reference SST" ; + dt_analysis:coordinates = "lon lat" ; + dt_analysis:long_name = "deviation from SST reference" ; + dt_analysis:source = "CMC0.1deg-CMC-L4-GLOB-v2.0" ; + dt_analysis:scale_factor = 0.1f ; + dt_analysis:units = "kelvin" ; + dt_analysis:valid_max = 127b ; + dt_analysis:valid_min = -127b ; + dt_analysis:_FillValue = -128b ; + dt_analysis:grid_mapping = "crs" ; + short l2p_flags(time, lat, lon) ; + l2p_flags:comment = "L2P common flags in bits 1-6 and data provider flags (from ACSPO mask) in bits 9-16: bit01 (0=IR: 1=microwave); bit02 (0=ocean; 1=land); bit03 (0=no ice; 1=ice); bits04-08 (reserved,set to 0); bit09 (0=radiance valid; 1=invalid); bit10 (0=night; 1=day); bit11 (0=ocean; 1=land); bit12 (0=good quality data; 1=degraded quality data due to \"twilight\" region); bit13 (0=no glint; 1=glint); bit14 (0=no snow/ice; 1=snow/ice); bits15-16 (00=clear; 01=probably clear; 10=cloudy; 11=clear-sky mask undefined)" ; + l2p_flags:coordinates = "lon lat" ; + l2p_flags:flag_masks = 1s, 2s, 4s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, -16384s ; + l2p_flags:flag_meanings = "microwave land ice invalid day land twilight glint ice probably_clear_or_cloudy_or_undefined" ; + l2p_flags:long_name = "L2P flags" ; + l2p_flags:valid_max = 32767s ; + l2p_flags:valid_min = -32768s ; + l2p_flags:grid_mapping = "crs" ; + float lat(lat) ; + lat:long_name = "latitude" ; + lat:comment = "Latitudes for locating data" ; + lat:units = "degrees_north" ; + lat:axis = "Y" ; + lat:valid_min = -90. ; + lat:valid_max = 90. ; + lat:standard_name = "latitude" ; + float lon(lon) ; + lon:long_name = "longitude" ; + lon:comment = "Longitude for locating data" ; + lon:units = "degrees_east" ; + lon:axis = "X" ; + lon:valid_min = -180. ; + lon:valid_max = 180. ; + lon:standard_name = "longitude" ; + short or_number_of_pixels(time, lat, lon) ; + or_number_of_pixels:_FillValue = 0s ; + or_number_of_pixels:long_name = "number of pixels from the L2Ps contributing to the SST value" ; + or_number_of_pixels:comment = "Original number of pixels from the L2Ps contributing to the SST value, not weighted" ; + or_number_of_pixels:units = "1" ; + or_number_of_pixels:coordinates = "lon lat" ; + or_number_of_pixels:valid_min = 1s ; + or_number_of_pixels:valid_max = 32767s ; + or_number_of_pixels:scale_factor = 1s ; + or_number_of_pixels:add_offset = 0s ; + or_number_of_pixels:grid_mapping = "crs" ; + byte quality_level(time, lat, lon) ; + quality_level:_FillValue = -128b ; + quality_level:valid_min = 0b ; + quality_level:valid_max = 5b ; + quality_level:long_name = "quality level of SST pixel" ; + quality_level:comment = "SST quality levels: 5 corresponds to “clear-sky” pixels and is recommended for operational applications and validation." ; + quality_level:coordinates = "lon lat" ; + quality_level:flag_meanings = "no_data bad_data not_used not_used not_used best_quality" ; + quality_level:flag_values = 1b, 2b, 3b, 4b, 5b ; + quality_level:grid_mapping = "crs" ; + byte satellite_zenith_angle(time, lat, lon) ; + satellite_zenith_angle:add_offset = 0.f ; + satellite_zenith_angle:comment = "satellite zenith angle" ; + satellite_zenith_angle:coordinates = "lon lat" ; + satellite_zenith_angle:long_name = "satellite zenith angle" ; + satellite_zenith_angle:scale_factor = 1.f ; + satellite_zenith_angle:units = "degrees" ; + satellite_zenith_angle:valid_max = 127b ; + satellite_zenith_angle:valid_min = -127b ; + satellite_zenith_angle:_FillValue = -128b ; + satellite_zenith_angle:grid_mapping = "crs" ; + short sea_surface_temperature(time, lat, lon) ; + sea_surface_temperature:add_offset = 273.15f ; + sea_surface_temperature:comment = "SST obtained by regression with buoy measurements, sensitive to skin SST. Further information at (Petrenko et al., JGR, 2014; doi:10.1002/2013JD020637)" ; + sea_surface_temperature:coordinates = "lon lat" ; + sea_surface_temperature:long_name = "sea surface sub-skin temperature" ; + sea_surface_temperature:scale_factor = 0.01f ; + sea_surface_temperature:source = "NOAA" ; + sea_surface_temperature:standard_name = "sea_surface_subskin_temperature" ; + sea_surface_temperature:units = "kelvin" ; + sea_surface_temperature:valid_max = 32767s ; + sea_surface_temperature:valid_min = -32767s ; + sea_surface_temperature:_FillValue = -32768s ; + sea_surface_temperature:grid_mapping = "crs" ; + byte sses_bias(time, lat, lon) ; + sses_bias:add_offset = 0.f ; + sses_bias:comment = "Bias is derived against Piecewise Regression SST produced by local regressions with buoys. Subtracting sses_bias from sea_surface_temperature produces more accurate estimate of SST at the depth of buoys. Further information at (Petrenko et al., JTECH, 2016; doi:10.1175/JTECH-D-15-0166.1)" ; + sses_bias:coordinates = "lon lat" ; + sses_bias:long_name = "SSES bias estimate" ; + sses_bias:scale_factor = 0.016f ; + sses_bias:units = "kelvin" ; + sses_bias:valid_max = 127b ; + sses_bias:valid_min = -127b ; + sses_bias:_FillValue = -128b ; + sses_bias:grid_mapping = "crs" ; + byte sses_standard_deviation(time, lat, lon) ; + sses_standard_deviation:add_offset = 1.f ; + sses_standard_deviation:comment = "Standard deviation of sea_surface_temperature from SST measured by drifting buoys. Further information at (Petrenko et al., JTECH, 2016; doi:10.1175/JTECH-D-15-0166.1)" ; + sses_standard_deviation:coordinates = "lon lat" ; + sses_standard_deviation:long_name = "SSES standard deviation" ; + sses_standard_deviation:scale_factor = 0.01f ; + sses_standard_deviation:units = "kelvin" ; + sses_standard_deviation:valid_max = 127b ; + sses_standard_deviation:valid_min = -127b ; + sses_standard_deviation:_FillValue = -128b ; + sses_standard_deviation:grid_mapping = "crs" ; + int sst_dtime(time, lat, lon) ; + sst_dtime:add_offset = 0.f ; + sst_dtime:comment = "time plus sst_dtime gives seconds since 1981-01-01 00:00:00 UTC" ; + sst_dtime:coordinates = "lon lat" ; + sst_dtime:long_name = "time difference from reference time" ; + sst_dtime:scale_factor = 0.25f ; + sst_dtime:units = "seconds" ; + sst_dtime:valid_max = 2147483647 ; + sst_dtime:valid_min = -2147483647 ; + sst_dtime:_FillValue = -2147483648 ; + sst_dtime:grid_mapping = "crs" ; + int time(time) ; + time:comment = "seconds since 1981-01-01 00:00:00" ; + time:long_name = "reference time of sst file" ; + time:standard_name = "time" ; + time:units = "seconds since 1981-01-01 00:00:00" ; + time:calendar = "Gregorian" ; + time:axis = "T" ; + byte wind_speed(time, lat, lon) ; + wind_speed:add_offset = 0.f ; + wind_speed:comment = "Typically represents surface winds (10 meters above the sea surface)" ; + wind_speed:coordinates = "lon lat" ; + wind_speed:height = "10 m" ; + wind_speed:long_name = "wind speed" ; + wind_speed:scale_factor = 0.15f ; + wind_speed:source = "Wind speed from NCEP GFS data" ; + wind_speed:standard_name = "wind_speed" ; + wind_speed:units = "m s-1" ; + wind_speed:valid_max = 127b ; + wind_speed:valid_min = -127b ; + wind_speed:_FillValue = -128b ; + wind_speed:grid_mapping = "crs" ; + +// global attributes: + :Conventions = "CF-1.6" ; + :Metadata_Conventions = "Unidata Dataset Discovery v1.0" ; + :acknowledgement = "Please acknowledge the use of these data with the following statement: These data were provided by Group for High Resolution Sea Surface Temperature (GHRSST) and the National Oceanic and Atmospheric Administration (NOAA)." ; + :cdm_data_type = "grid" ; + :comment = "SSTs are a weighted average of the SSTs of contributing pixels. WARNING: some applications are unable to properly handle signed byte values. If byte values > 127 are encountered, subtract 256 from this reported value." ; + :creator_email = "Alex.Ignatov@noaa.gov" ; + :creator_name = "Alex Ignatov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :date_created = "20210701T013122Z" ; + :easternmost_longitude = -179.98f ; + :file_quality_level = 2 ; + :gds_version_id = "02.0" ; + :geospatial_lat_max = -45.44f ; + :geospatial_lat_min = -87.f ; + :geospatial_lat_resolution = 0.02f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_max = -179.98f ; + :geospatial_lon_min = -180.f ; + :geospatial_lon_resolution = 0.02f ; + :geospatial_lon_units = "degrees_east" ; + :history = "Mon Sep 25 18:37:47 2023: ncks -d lon,15575,15580 -d lat,7270,7275 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/sst/20210701000000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc sst_MB_202107010000.nc\nCreated by the L2P-to-L3U conversion tool, which was developed and provided by NOAA/NESDIS/STAR and CCNY. The version is 4.1.5" ; + :id = "AVHRRF_MB-OSPO-L3U-v2.7" ; + :institution = "NOAA/NESDIS/OSPO" ; + :keywords = "Oceans > Ocean Temperature > Sea Surface Temperature" ; + :keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Science Keywords" ; + :license = "GHRSST protocol describes data use as free and open" ; + :metadata_link = "http://podaac.jpl.nasa.gov/ws/metadata/dataset/?format=iso&shortName=AVHRRF_MB-OSPO-L2P-v2.7" ; + :naming_authority = "org.ghrsst" ; + :netcdf_version_id = "4.5.0 of Jul 23 2018 18:00:31 $" ; + :northernmost_latitude = -45.44f ; + :platform = "MetOpB" ; + :processing_level = "L3U" ; + :product_version = "L2P algorithm V2.70; L3U algorithm V4.1.5" ; + :project = "Group for High Resolution Sea Surface Temperature" ; + :publisher_email = "ghrsst-po@nceo.ac.uk" ; + :publisher_name = "The GHRSST Project Office" ; + :publisher_url = "http://www.ghrsst.org" ; + :references = "Data convention: GHRSST Data Specification (GDS) v2.0. Algorithms: ACSPO-AVHRR ATBD (NOAA/NESDIS/OSPO)" ; + :sensor = "AVHRR" ; + :source = "l2p_source : 20210701000000-OSPO-L2P_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc" ; + :southernmost_latitude = -87.f ; + :spatial_resolution = "0.02 deg" ; + :standard_name_vocabulary = "CF Standard Name Table (v26, 08 November 2013)" ; + :start_time = "20210701T000000Z" ; + :stop_time = "20210701T000959Z" ; + :summary = "Sea surface temperature retrievals produced by NOAA/NESDIS/OSPO office from AVHRR sensor" ; + :time_coverage_end = "20210701T000959Z" ; + :time_coverage_start = "20210701T000000Z" ; + :title = "AVHRR L3U SST" ; + :uuid = "2b7fc1f5-6c67-462d-b9c5-b9e9a22b2a33" ; + :westernmost_longitude = -180.f ; + :sst_luts = "LUT_AVHRRF_MB_L2P_DEPTH_DAY_V01.01_20180507.txt, LUT_AVHRRF_MB_L2P_SKIN_DAYNIGHT_V01.00_20180408.txt, LUT_AVHRRF_MB_L2P_DEPTH_NIGHT_V01.01_20180507.txt, LUT_AVHRRF_MB_L2P_SKIN_NIGHT_V01.00_20180408.txt" ; + :geospatial_bounds = "POLYGON(( 58.354 -67.757, 113.644 -45.517, 152.834 -52.600, -163.078 -84.514, 58.354 -67.757))" ; + :row_start = 6772 ; + :row_count = 2077 ; + :col_start = 0 ; + :col_count = 18000 ; + :NCO = "netCDF Operators version 4.9.3 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + crs = 0 ; + + dt_analysis = + 5, 5, 5, 4, 4, 4, + 5, 6, 6, 6, 5, 4, + 6, 6, 6, 5, 5, 5, + 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 6, 6, 6, 6, 6 ; + + l2p_flags = + 2560, 2560, 2560, 2560, 2560, 2560, + 2560, 2560, 2560, 2560, 2560, 2560, + 2560, 2560, 2560, 2560, 2560, 2560, + 2560, 2560, 2560, 2560, 2560, 2560, + 2560, 2560, 2560, 2560, 2560, 2560, + 2560, 2560, 2560, 2560, 2560, 2560 ; + + lat = -55.41, -55.43, -55.45, -55.47, -55.49, -55.51 ; + + lon = 131.51, 131.53, 131.55, 131.57, 131.59, 131.61 ; + + or_number_of_pixels = + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11 ; + + quality_level = + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5 ; + + satellite_zenith_angle = + -9, -9, -9, -9, -10, -10, + -9, -9, -9, -10, -10, -10, + -9, -9, -9, -10, -10, -10, + -9, -9, -10, -10, -10, -10, + -9, -9, -10, -10, -10, -10, + -9, -10, -10, -10, -10, -10 ; + + sea_surface_temperature = + 374, 377, 378, 377, 379, 378, + 383, 389, 391, 389, 383, 379, + 391, 392, 390, 388, 388, 383, + 388, 388, 390, 384, 384, 382, + 382, 382, 383, 387, 386, 387, + 383, 389, 390, 391, 392, 392 ; + + sses_bias = + 9, 10, 10, 9, 9, 9, + 10, 11, 11, 11, 10, 10, + 12, 11, 11, 11, 11, 10, + 11, 11, 10, 11, 11, 11, + 12, 11, 11, 11, 11, 12, + 12, 11, 11, 11, 11, 11 ; + + sses_standard_deviation = + -68, -68, -68, -68, -68, -68, + -68, -68, -68, -68, -68, -68, + -68, -68, -68, -68, -68, -68, + -68, -68, -68, -68, -68, -68, + -68, -68, -68, -68, -68, -68, + -68, -68, -68, -68, -68, -68 ; + + sst_dtime = + 307, 307, 306, 306, 306, 306, + 308, 308, 308, 307, 307, 307, + 309, 309, 309, 309, 309, 308, + 311, 311, 310, 310, 310, 310, + 312, 312, 311, 311, 311, 311, + 313, 313, 313, 313, 312, 312 ; + + time = 1277942400 ; + + wind_speed = + 52, 51, 51, 51, 51, 51, + 52, 52, 52, 51, 51, 51, + 52, 52, 52, 52, 52, 51, + 53, 52, 52, 52, 52, 52, + 53, 53, 52, 52, 52, 52, + 53, 53, 53, 53, 52, 52 ; +} diff --git a/utils/test/testdata/ghrsst_sst_mb_202107010100.cdl b/utils/test/testdata/ghrsst_sst_mb_202107010100.cdl new file mode 100644 index 000000000..fed17ed78 --- /dev/null +++ b/utils/test/testdata/ghrsst_sst_mb_202107010100.cdl @@ -0,0 +1,295 @@ +netcdf sst_MB_202107010100 { +dimensions: + time = 1 ; + lat = 6 ; + lon = 6 ; +variables: + int crs ; + crs:grid_mapping_name = "latitude_longitude" ; + crs:horizontal_datum_name = "World Geodetic System 1984" ; + byte dt_analysis(time, lat, lon) ; + dt_analysis:add_offset = 0.f ; + dt_analysis:comment = "Deviation from reference SST, i.e., dt_analysis = SST - reference SST" ; + dt_analysis:coordinates = "lon lat" ; + dt_analysis:long_name = "deviation from SST reference" ; + dt_analysis:source = "CMC0.1deg-CMC-L4-GLOB-v2.0" ; + dt_analysis:scale_factor = 0.1f ; + dt_analysis:units = "kelvin" ; + dt_analysis:valid_max = 127b ; + dt_analysis:valid_min = -127b ; + dt_analysis:_FillValue = -128b ; + dt_analysis:grid_mapping = "crs" ; + short l2p_flags(time, lat, lon) ; + l2p_flags:comment = "L2P common flags in bits 1-6 and data provider flags (from ACSPO mask) in bits 9-16: bit01 (0=IR: 1=microwave); bit02 (0=ocean; 1=land); bit03 (0=no ice; 1=ice); bits04-08 (reserved,set to 0); bit09 (0=radiance valid; 1=invalid); bit10 (0=night; 1=day); bit11 (0=ocean; 1=land); bit12 (0=good quality data; 1=degraded quality data due to \"twilight\" region); bit13 (0=no glint; 1=glint); bit14 (0=no snow/ice; 1=snow/ice); bits15-16 (00=clear; 01=probably clear; 10=cloudy; 11=clear-sky mask undefined)" ; + l2p_flags:coordinates = "lon lat" ; + l2p_flags:flag_masks = 1s, 2s, 4s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, -16384s ; + l2p_flags:flag_meanings = "microwave land ice invalid day land twilight glint ice probably_clear_or_cloudy_or_undefined" ; + l2p_flags:long_name = "L2P flags" ; + l2p_flags:valid_max = 32767s ; + l2p_flags:valid_min = -32768s ; + l2p_flags:grid_mapping = "crs" ; + float lat(lat) ; + lat:long_name = "latitude" ; + lat:comment = "Latitudes for locating data" ; + lat:units = "degrees_north" ; + lat:axis = "Y" ; + lat:valid_min = -90. ; + lat:valid_max = 90. ; + lat:standard_name = "latitude" ; + float lon(lon) ; + lon:long_name = "longitude" ; + lon:comment = "Longitude for locating data" ; + lon:units = "degrees_east" ; + lon:axis = "X" ; + lon:valid_min = -180. ; + lon:valid_max = 180. ; + lon:standard_name = "longitude" ; + short or_number_of_pixels(time, lat, lon) ; + or_number_of_pixels:_FillValue = 0s ; + or_number_of_pixels:long_name = "number of pixels from the L2Ps contributing to the SST value" ; + or_number_of_pixels:comment = "Original number of pixels from the L2Ps contributing to the SST value, not weighted" ; + or_number_of_pixels:units = "1" ; + or_number_of_pixels:coordinates = "lon lat" ; + or_number_of_pixels:valid_min = 1s ; + or_number_of_pixels:valid_max = 32767s ; + or_number_of_pixels:scale_factor = 1s ; + or_number_of_pixels:add_offset = 0s ; + or_number_of_pixels:grid_mapping = "crs" ; + byte quality_level(time, lat, lon) ; + quality_level:_FillValue = -128b ; + quality_level:valid_min = 0b ; + quality_level:valid_max = 5b ; + quality_level:long_name = "quality level of SST pixel" ; + quality_level:comment = "SST quality levels: 5 corresponds to “clear-sky” pixels and is recommended for operational applications and validation." ; + quality_level:coordinates = "lon lat" ; + quality_level:flag_meanings = "no_data bad_data not_used not_used not_used best_quality" ; + quality_level:flag_values = 1b, 2b, 3b, 4b, 5b ; + quality_level:grid_mapping = "crs" ; + byte satellite_zenith_angle(time, lat, lon) ; + satellite_zenith_angle:add_offset = 0.f ; + satellite_zenith_angle:comment = "satellite zenith angle" ; + satellite_zenith_angle:coordinates = "lon lat" ; + satellite_zenith_angle:long_name = "satellite zenith angle" ; + satellite_zenith_angle:scale_factor = 1.f ; + satellite_zenith_angle:units = "degrees" ; + satellite_zenith_angle:valid_max = 127b ; + satellite_zenith_angle:valid_min = -127b ; + satellite_zenith_angle:_FillValue = -128b ; + satellite_zenith_angle:grid_mapping = "crs" ; + short sea_surface_temperature(time, lat, lon) ; + sea_surface_temperature:add_offset = 273.15f ; + sea_surface_temperature:comment = "SST obtained by regression with buoy measurements, sensitive to skin SST. Further information at (Petrenko et al., JGR, 2014; doi:10.1002/2013JD020637)" ; + sea_surface_temperature:coordinates = "lon lat" ; + sea_surface_temperature:long_name = "sea surface sub-skin temperature" ; + sea_surface_temperature:scale_factor = 0.01f ; + sea_surface_temperature:source = "NOAA" ; + sea_surface_temperature:standard_name = "sea_surface_subskin_temperature" ; + sea_surface_temperature:units = "kelvin" ; + sea_surface_temperature:valid_max = 32767s ; + sea_surface_temperature:valid_min = -32767s ; + sea_surface_temperature:_FillValue = -32768s ; + sea_surface_temperature:grid_mapping = "crs" ; + byte sses_bias(time, lat, lon) ; + sses_bias:add_offset = 0.f ; + sses_bias:comment = "Bias is derived against Piecewise Regression SST produced by local regressions with buoys. Subtracting sses_bias from sea_surface_temperature produces more accurate estimate of SST at the depth of buoys. Further information at (Petrenko et al., JTECH, 2016; doi:10.1175/JTECH-D-15-0166.1)" ; + sses_bias:coordinates = "lon lat" ; + sses_bias:long_name = "SSES bias estimate" ; + sses_bias:scale_factor = 0.016f ; + sses_bias:units = "kelvin" ; + sses_bias:valid_max = 127b ; + sses_bias:valid_min = -127b ; + sses_bias:_FillValue = -128b ; + sses_bias:grid_mapping = "crs" ; + byte sses_standard_deviation(time, lat, lon) ; + sses_standard_deviation:add_offset = 1.f ; + sses_standard_deviation:comment = "Standard deviation of sea_surface_temperature from SST measured by drifting buoys. Further information at (Petrenko et al., JTECH, 2016; doi:10.1175/JTECH-D-15-0166.1)" ; + sses_standard_deviation:coordinates = "lon lat" ; + sses_standard_deviation:long_name = "SSES standard deviation" ; + sses_standard_deviation:scale_factor = 0.01f ; + sses_standard_deviation:units = "kelvin" ; + sses_standard_deviation:valid_max = 127b ; + sses_standard_deviation:valid_min = -127b ; + sses_standard_deviation:_FillValue = -128b ; + sses_standard_deviation:grid_mapping = "crs" ; + int sst_dtime(time, lat, lon) ; + sst_dtime:add_offset = 0.f ; + sst_dtime:comment = "time plus sst_dtime gives seconds since 1981-01-01 00:00:00 UTC" ; + sst_dtime:coordinates = "lon lat" ; + sst_dtime:long_name = "time difference from reference time" ; + sst_dtime:scale_factor = 0.25f ; + sst_dtime:units = "seconds" ; + sst_dtime:valid_max = 2147483647 ; + sst_dtime:valid_min = -2147483647 ; + sst_dtime:_FillValue = -2147483648 ; + sst_dtime:grid_mapping = "crs" ; + int time(time) ; + time:comment = "seconds since 1981-01-01 00:00:00" ; + time:long_name = "reference time of sst file" ; + time:standard_name = "time" ; + time:units = "seconds since 1981-01-01 00:00:00" ; + time:calendar = "Gregorian" ; + time:axis = "T" ; + byte wind_speed(time, lat, lon) ; + wind_speed:add_offset = 0.f ; + wind_speed:comment = "Typically represents surface winds (10 meters above the sea surface)" ; + wind_speed:coordinates = "lon lat" ; + wind_speed:height = "10 m" ; + wind_speed:long_name = "wind speed" ; + wind_speed:scale_factor = 0.15f ; + wind_speed:source = "Wind speed from NCEP GFS data" ; + wind_speed:standard_name = "wind_speed" ; + wind_speed:units = "m s-1" ; + wind_speed:valid_max = 127b ; + wind_speed:valid_min = -127b ; + wind_speed:_FillValue = -128b ; + wind_speed:grid_mapping = "crs" ; + +// global attributes: + :Conventions = "CF-1.6" ; + :Metadata_Conventions = "Unidata Dataset Discovery v1.0" ; + :acknowledgement = "Please acknowledge the use of these data with the following statement: These data were provided by Group for High Resolution Sea Surface Temperature (GHRSST) and the National Oceanic and Atmospheric Administration (NOAA)." ; + :cdm_data_type = "grid" ; + :comment = "SSTs are a weighted average of the SSTs of contributing pixels. WARNING: some applications are unable to properly handle signed byte values. If byte values > 127 are encountered, subtract 256 from this reported value." ; + :creator_email = "Alex.Ignatov@noaa.gov" ; + :creator_name = "Alex Ignatov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :date_created = "20210701T021440Z" ; + :easternmost_longitude = -179.98f ; + :file_quality_level = 2 ; + :gds_version_id = "02.0" ; + :geospatial_lat_max = 89.f ; + :geospatial_lat_min = 52.9f ; + :geospatial_lat_resolution = 0.02f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_max = -179.98f ; + :geospatial_lon_min = -180.f ; + :geospatial_lon_resolution = 0.02f ; + :geospatial_lon_units = "degrees_east" ; + :history = "Mon Sep 25 18:37:49 2023: ncks -d lon,16555,16560 -d lat,1664,1669 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/sst/20210701010000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc sst_MB_202107010100.nc\nCreated by the L2P-to-L3U conversion tool, which was developed and provided by NOAA/NESDIS/STAR and CCNY. The version is 4.1.5" ; + :id = "AVHRRF_MB-OSPO-L3U-v2.7" ; + :institution = "NOAA/NESDIS/OSPO" ; + :keywords = "Oceans > Ocean Temperature > Sea Surface Temperature" ; + :keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Science Keywords" ; + :license = "GHRSST protocol describes data use as free and open" ; + :metadata_link = "http://podaac.jpl.nasa.gov/ws/metadata/dataset/?format=iso&shortName=AVHRRF_MB-OSPO-L2P-v2.7" ; + :naming_authority = "org.ghrsst" ; + :netcdf_version_id = "4.5.0 of Jul 23 2018 18:00:31 $" ; + :northernmost_latitude = 89.f ; + :platform = "MetOpB" ; + :processing_level = "L3U" ; + :product_version = "L2P algorithm V2.70; L3U algorithm V4.1.5" ; + :project = "Group for High Resolution Sea Surface Temperature" ; + :publisher_email = "ghrsst-po@nceo.ac.uk" ; + :publisher_name = "The GHRSST Project Office" ; + :publisher_url = "http://www.ghrsst.org" ; + :references = "Data convention: GHRSST Data Specification (GDS) v2.0. Algorithms: ACSPO-AVHRR ATBD (NOAA/NESDIS/OSPO)" ; + :sensor = "AVHRR" ; + :source = "l2p_source : 20210701010000-OSPO-L2P_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc" ; + :southernmost_latitude = 52.9f ; + :spatial_resolution = "0.02 deg" ; + :standard_name_vocabulary = "CF Standard Name Table (v26, 08 November 2013)" ; + :start_time = "20210701T010000Z" ; + :stop_time = "20210701T010959Z" ; + :summary = "Sea surface temperature retrievals produced by NOAA/NESDIS/OSPO office from AVHRR sensor" ; + :time_coverage_end = "20210701T010959Z" ; + :time_coverage_start = "20210701T010000Z" ; + :title = "AVHRR L3U SST" ; + :uuid = "02508803-88b0-49c3-8a26-5a4a0f33099a" ; + :westernmost_longitude = -180.f ; + :sst_luts = "LUT_AVHRRF_MB_L2P_DEPTH_DAY_V01.01_20180507.txt, LUT_AVHRRF_MB_L2P_SKIN_DAYNIGHT_V01.00_20180408.txt, LUT_AVHRRF_MB_L2P_DEPTH_NIGHT_V01.01_20180507.txt, LUT_AVHRRF_MB_L2P_SKIN_NIGHT_V01.00_20180408.txt" ; + :geospatial_bounds = "POLYGON(( 114.037 60.830, -12.301 83.131, -127.738 67.699, 160.334 52.991, 114.037 60.830))" ; + :row_start = 50 ; + :row_count = 1804 ; + :col_start = 0 ; + :col_count = 18000 ; + :NCO = "netCDF Operators version 4.9.3 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + crs = 0 ; + + dt_analysis = + 7, 7, 7, 7, 7, 7, + 7, 7, 8, 8, 7, 7, + 8, 8, 7, 7, 7, 7, + 8, 8, 7, 7, 7, 8, + 8, 7, 7, 7, 7, 8, + 8, 8, 8, 7, 7, 8 ; + + l2p_flags = + 4608, 4608, 4608, 4608, 4608, 4608, + 4608, 4608, 4608, 4608, 4608, 4608, + 4608, 4608, 4608, 4608, 4608, 4608, + 4608, 4608, 4608, 4608, 4608, 4608, + 4608, 4608, 4608, 4608, 4608, 4608, + 4608, 4608, 4608, 4608, 4608, 4608 ; + + lat = 56.71, 56.69, 56.67, 56.65, 56.63, 56.61 ; + + lon = 151.11, 151.13, 151.15, 151.17, 151.19, 151.21 ; + + or_number_of_pixels = + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11 ; + + quality_level = + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5 ; + + satellite_zenith_angle = + -46, -46, -46, -46, -46, -46, + -46, -46, -46, -46, -46, -46, + -46, -46, -46, -46, -46, -46, + -46, -46, -46, -46, -46, -46, + -46, -46, -46, -46, -46, -46, + -46, -46, -46, -46, -46, -46 ; + + sea_surface_temperature = + 982, 981, 981, 979, 975, 972, + 982, 981, 979, 977, 973, 972, + 982, 980, 974, 969, 969, 970, + 982, 979, 972, 970, 970, 971, + 974, 972, 969, 967, 968, 969, + 970, 970, 968, 966, 965, 968 ; + + sses_bias = + 83, 82, 80, 79, 78, 77, + 81, 80, 79, 79, 78, 77, + 78, 78, 78, 78, 78, 76, + 77, 78, 78, 77, 76, 75, + 78, 78, 78, 76, 75, 74, + 77, 78, 77, 75, 74, 75 ; + + sses_standard_deviation = + -39, -39, -39, -39, -39, -39, + -40, -40, -39, -39, -39, -39, + -42, -41, -40, -39, -39, -39, + -44, -41, -39, -39, -39, -40, + -41, -40, -39, -39, -40, -40, + -40, -39, -39, -42, -43, -45 ; + + sst_dtime = + 2382, 2382, 2381, 2381, 2381, 2380, + 2383, 2383, 2382, 2382, 2382, 2381, + 2384, 2384, 2384, 2383, 2383, 2383, + 2386, 2385, 2385, 2384, 2384, 2384, + 2387, 2386, 2386, 2386, 2385, 2385, + 2388, 2388, 2387, 2387, 2386, 2386 ; + + time = 1277946000 ; + + wind_speed = + 43, 44, 44, 44, 44, 45, + 44, 44, 44, 45, 45, 45, + 44, 44, 45, 45, 45, 45, + 45, 45, 45, 45, 45, 46, + 45, 45, 45, 46, 46, 46, + 45, 45, 46, 46, 46, 46 ; +} diff --git a/utils/test/testinput/gdas_rads2ioda.yaml b/utils/test/testinput/gdas_rads2ioda.yaml index c2daa9a3b..97748444d 100644 --- a/utils/test/testinput/gdas_rads2ioda.yaml +++ b/utils/test/testinput/gdas_rads2ioda.yaml @@ -5,3 +5,4 @@ variable: absoluteDynamicTopography output file: rads_adt_j3_2021182.ioda.nc input files: - rads_adt_j3_2021182.nc4 +- rads_adt_j3_2021182.nc4