Add JPSSRR sea-ice product to ioda converter (#1259)

#### Description - Task for adding a new ioda converter for new sea-ice concentration - For sea-ice, AMSR2, MIRS and JPSSRR product will be used Partially addressed #1182 --------- Co-authored-by: Guillaume Vernieres <[email protected]>
NOAA-EMC · Aug 27, 2024 · 419c21f · 419c21f
1 parent 09594d1
commit 419c21f
Show file tree

Hide file tree

Showing 8 changed files with 1,686 additions and 0 deletions.
diff --git a/utils/obsproc/IcecJpssrr2Ioda.h b/utils/obsproc/IcecJpssrr2Ioda.h
@@ -0,0 +1,125 @@
+#pragma once
+
+#include <ctime>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <netcdf>    // NOLINT (using C API)
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "eckit/config/LocalConfiguration.h"
+
+#include <Eigen/Dense>    // NOLINT
+
+#include "ioda/../../../../core/IodaUtils.h"
+#include "ioda/Group.h"
+#include "ioda/ObsGroup.h"
+
+#include "oops/util/dateFunctions.h"
+
+#include "NetCDFToIodaConverter.h"
+
+namespace gdasapp {
+
+  class IcecJpssrr2Ioda : public NetCDFToIodaConverter {
+   public:
+    explicit IcecJpssrr2Ioda(const eckit::Configuration & fullConfig, const eckit::mpi::Comm & comm)
+      : NetCDFToIodaConverter(fullConfig, comm) {
+      variable_ = "seaIceFraction";
+    }
+
+    // Read netcdf file and populate iodaVars
+    gdasapp::obsproc::iodavars::IodaVars providerToIodaVars(const std::string fileName) final {
+      oops::Log::info() << "Processing files provided by the JPSSRR" << std::endl;
+
+      // Open the NetCDF file in read-only mode
+      netCDF::NcFile ncFile(fileName, netCDF::NcFile::read);
+      oops::Log::info() << "Reading... " << fileName << std::endl;
+
+      // Get the number of obs in the file
+      int dimxSize = ncFile.getDim("Columns").getSize();
+      int dimySize = ncFile.getDim("Rows").getSize();
+      int nobs = dimxSize * dimySize;
+
+      // Set the int metadata names
+      std::vector<std::string> intMetadataNames = {"oceanBasin"};
+
+      // Set the float metadata name
+      std::vector<std::string> floatMetadataNames = {};
+
+      // Create instance of iodaVars object
+      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, floatMetadataNames, intMetadataNames);
+
+      oops::Log::debug() << "--- iodaVars.location_: " << iodaVars.location_ << std::endl;
+
+      // Read non-optional metadata: longitude and latitude
+      std::vector<float> oneDimLatVal(iodaVars.location_);
+      ncFile.getVar("Latitude").getVar(oneDimLatVal.data());
+
+      std::vector<float> oneDimLonVal(iodaVars.location_);
+      ncFile.getVar("Longitude").getVar(oneDimLonVal.data());
+
+      // Create a vector to hold the Summary Qc variable
+      // User-level summary QC: 0=Normal, 1=Uncertain
+      std::vector<signed char> oneDimFlagsVal(iodaVars.location_);
+      ncFile.getVar("SummaryQC_Ice_Concentration").getVar(oneDimFlagsVal.data());
+
+      // Get Ice_Concentration obs values
+      std::vector<float> oneDimObsVal(iodaVars.location_);
+      ncFile.getVar("IceConc").getVar(oneDimObsVal.data());
+
+      // Read and process the starting and ending of dateTime
+      auto timeStartAttr = ncFile.getAtt("time_coverage_start");
+      auto timeEndAttr = ncFile.getAtt("time_coverage_end");
+
+      // Extract attribute values as strings
+      std::string timeStartStr, timeEndStr;
+      timeStartAttr.getValues(timeStartStr);
+      timeEndAttr.getValues(timeEndStr);
+
+      // Convert the extracted strings to util::DateTime objects
+      util::DateTime timeStartDtime(timeStartStr);
+      util::DateTime timeEndDtime(timeEndStr);
+
+      // Create vectors of util::DateTime
+      std::vector<util::DateTime> timeStartVector = {timeStartDtime};
+      std::vector<util::DateTime> timeEndVector = {timeEndDtime};
+
+      // Set epoch time for JPSSRR ICEC
+      util::DateTime epochDtime("1970-01-01T00:00:00Z");
+
+      // Convert Obs DateTime objects to epoch time offsets in seconds
+      int64_t timeStartOffsets
+         = ioda::convertDtimeToTimeOffsets(epochDtime, timeStartVector)[0];
+      int64_t timeEndOffsets
+         = ioda::convertDtimeToTimeOffsets(epochDtime, timeEndVector)[0];
+
+      iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z";
+
+      // Update non-optional Eigen arrays
+      for (int i = 0; i < iodaVars.location_; i++) {
+        iodaVars.longitude_(i) = oneDimLonVal[i];
+        iodaVars.latitude_(i) = oneDimLatVal[i];
+        iodaVars.obsVal_(i) = static_cast<float>(oneDimObsVal[i]*0.01);
+        iodaVars.obsError_(i) = 0.1;  // Do something for obs error
+        iodaVars.preQc_(i) = oneDimFlagsVal[i];
+        iodaVars.datetime_(i) =
+            timeStartOffsets + (timeEndOffsets - timeStartOffsets) * 0.5;
+        // Store optional metadata, set ocean basins to -999 for now
+        iodaVars.intMetadata_.row(i) << -999;
+      }
+
+      // basic test for iodaVars.trim
+      Eigen::Array<bool, Eigen::Dynamic, 1> mask =
+          ((iodaVars.obsVal_ >= 0.0 && iodaVars.obsVal_ <= 1.0) &&
+           iodaVars.datetime_ > 0.0 &&
+          (iodaVars.latitude_ <= -40.0 || iodaVars.latitude_ >= 40.0));
+      iodaVars.trim(mask);
+
+      return iodaVars;
+    };
+  };  // class IcecMirs2Ioda
+}  // namespace gdasapp
+
diff --git a/utils/obsproc/applications/gdas_obsprovider2ioda.h b/utils/obsproc/applications/gdas_obsprovider2ioda.h
@@ -8,6 +8,7 @@
 
 #include "../Ghrsst2Ioda.h"
 #include "../IcecAmsr2Ioda.h"
+#include "../IcecJpssrr2Ioda.h"
 #include "../IcecMirs2Ioda.h"
 #include "../Rads2Ioda.h"
 #include "../RTOFSSalinity.h"
@@ -54,6 +55,9 @@ namespace gdasapp {
       } else if (provider == "MIRS") {
         IcecMirs2Ioda conv2ioda(fullConfig, this->getComm());
         conv2ioda.writeToIoda();
+      } else if (provider == "JPSSRR") {
+        IcecJpssrr2Ioda conv2ioda(fullConfig, this->getComm());
+        conv2ioda.writeToIoda();
       } else if (provider == "VIIRSAOD") {
         Viirsaod2Ioda conv2ioda(fullConfig, this->getComm());
         conv2ioda.writeToIoda();

diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt
@@ -9,6 +9,7 @@ list( APPEND utils_test_input
   testinput/gdas_smos2ioda.yaml
   testinput/gdas_icecamsr2ioda.yaml
   testinput/gdas_icecmirs2ioda.yaml
+  testinput/gdas_icecjpssrr2ioda.yaml
   testinput/gdas_viirsaod2ioda.yaml
 )
 
@@ -21,6 +22,7 @@ set( gdas_utils_test_ref
   testref/smos2ioda.test
   testref/icecamsr2ioda.test
   testref/icecmirs2ioda.test
+  testref/icecjpssrr2ioda.test
   testref/viirsaod2ioda.test
 )
 
@@ -154,3 +156,10 @@ ecbuild_add_test( TARGET  test_gdasapp_util_icecmirs2ioda
                   ARGS    "../testinput/gdas_icecmirs2ioda.yaml"
                   LIBS    gdas-utils
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
+
+	  # Test the JPSSRR to IODA converter
+ecbuild_add_test( TARGET  test_gdasapp_util_icecjpssrr2ioda
+                  COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x
+                  ARGS    "../testinput/gdas_icecjpssrr2ioda.yaml"
+                  LIBS    gdas-utils
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc)
diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh
@@ -25,6 +25,8 @@ cdl2nc4 icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1
 cdl2nc4 icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl
 cdl2nc4 icec_mirs_snpp_1.nc4 ${project_source_dir}/testdata/icec_mirs_snpp_1.cdl
 cdl2nc4 icec_mirs_snpp_2.nc4 ${project_source_dir}/testdata/icec_mirs_snpp_2.cdl
+cdl2nc4 icec_jrr_n20_1.nc4 ${project_source_dir}/testdata/icec_jrr_n20_1.cdl
+cdl2nc4 icec_jrr_n20_2.nc4 ${project_source_dir}/testdata/icec_jrr_n20_2.cdl
 cdl2nc4 sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl
 cdl2nc4 sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl
 cdl2nc4 sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl