diff --git a/.gitignore b/.gitignore index decc5f338..bc5569486 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ exp.summary.txt CMakeLists.txt.user dependencies.mk loadmap +git-hammer-config.json #Ignore the contents of these directories #(html and latex directories are generated by doxygen) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96bcfbe85..3e6c01022 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ ################################################################################ cmake_minimum_required( VERSION 3.12 ) -project( oops VERSION 1.1.0 LANGUAGES CXX Fortran ) +project( oops VERSION 1.2.0 LANGUAGES CXX Fortran ) ## Ecbuild integration find_package( ecbuild 3.3.2 REQUIRED ) diff --git a/LICENSE b/LICENSE.md similarity index 99% rename from LICENSE rename to LICENSE.md index f9d26d858..a262326ad 100644 --- a/LICENSE +++ b/LICENSE.md @@ -188,7 +188,7 @@ identification within third-party archives. Copyright 2009-2016 European Centre for Medium-Range Weather Forecasts (ECMWF) - Copyright 2017-2020 University Corporation for Atmospheric Research (UCAR) + Copyright 2017-2021 University Corporation for Atmospheric Research (UCAR) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 4d44a2931..7c7fce81a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ Object Oriented Prediction System (OOPS) -GNU: [![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiN2RzbkhSR3o4dm9HN29wRW5laDRMdGlSdGFrNUswWE9pMVJHbmwrQ0EvYWZkNnRjMTFzZWFQL3dUdUpLbmUxNlJXYlFsYmRoaDR5K1NUdCtWNmNvTHFBPSIsIml2UGFyYW1ldGVyU3BlYyI6ImV5OWJFdFRWNWVnMVpwcHEiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-gnu/history?region=us-east-1) -INTEL: [![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiWG5KM0dQR3VBNWdVVUdJeURwUjhoOXNWYlBYdnpGR2prLzQ4RXk2ZlkrN2llZ2M4S05MeXdWcERMVFZ2RGIxbTRXZFMvTFM5cTRoamFzV3hYanlZVEVNPSIsIml2UGFyYW1ldGVyU3BlYyI6IjZJVnJYL1Z5VktTQWZzZUkiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-intel) -CLANG: [![AWS-clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiNHdvREpKTnVOT2pneEx6cHpaMjlma3RsUmlLZFQrMmpDR0tNODBrMzA0Z0pFazFKMFloQVc0Q0xkeGRZbW9WSDJ4c2FWSFg0SkZWb1VXVmVQRk5rSTFJPSIsIml2UGFyYW1ldGVyU3BlYyI6InRQckQrWDdaY0l2RkZoWmUiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-clang/history?region=us-east-1) -[![codecov](https://codecov.io/gh/JCSDA/oops/branch/develop/graph/badge.svg?token=GdDzbEQedm)](https://codecov.io/gh/JCSDA/oops) +### Continuous integration: +| Platform | JCSDA-internal | JCSDA | +| ------------- | ------------- |------------- | +| GNU | [![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiNjVsdE5keGM1Tk5yV3FEd2tiaU5YRFVLWEVUTitBdCs5dm9nc29LNjh0T1plank1bzB2TTB2ZkptT2lUSFNWMklmWWdkNW5IbFR1VXZ1NE12M0JiR3BrPSIsIml2UGFyYW1ldGVyU3BlYyI6IkthTWxpTkZaY21CUklINnciLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/oops-internal-gnu/history) | [![AWS-gnu](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiN2RzbkhSR3o4dm9HN29wRW5laDRMdGlSdGFrNUswWE9pMVJHbmwrQ0EvYWZkNnRjMTFzZWFQL3dUdUpLbmUxNlJXYlFsYmRoaDR5K1NUdCtWNmNvTHFBPSIsIml2UGFyYW1ldGVyU3BlYyI6ImV5OWJFdFRWNWVnMVpwcHEiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://us-east-1.console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-gnu/history) +| Intel | [![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiUkd1YnZqN2RLWWhaOFE5MkdmbGZ5K1I0amlGU3l4Z3h5Ymw2KzR1Z3lhMXNFSEpkK2M2QTlIZk9mdVQ0ZEx5UFMvRDBZa2tUbVd6TkU1QmZPRWEycmNRPSIsIml2UGFyYW1ldGVyU3BlYyI6IjhmY2tER1c5SDVYekJZRS8iLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/oops-internal-intel/history) | [![AWS-intel](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiWG5KM0dQR3VBNWdVVUdJeURwUjhoOXNWYlBYdnpGR2prLzQ4RXk2ZlkrN2llZ2M4S05MeXdWcERMVFZ2RGIxbTRXZFMvTFM5cTRoamFzV3hYanlZVEVNPSIsIml2UGFyYW1ldGVyU3BlYyI6IjZJVnJYL1Z5VktTQWZzZUkiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://us-east-1.console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-intel/history) +| CLANG | [![AWS-clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiL3F2OHVEcmc0WitOTlY3Vm9LV0J3NHREZ0RicUZ6RDFXcFNlODhrRHN6a0RjNlF2TDI0VzM2S0VBKzMxR1BIREdYNmZrRWRUcm9xd0h2ZnhYNzVNc3NJPSIsIml2UGFyYW1ldGVyU3BlYyI6Iit1Y01tZSt4b1VtbGxKeDQiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://console.aws.amazon.com/codesuite/codebuild/469205354006/projects/oops-internal-clang/history) | [![AWS-clang](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiNHdvREpKTnVOT2pneEx6cHpaMjlma3RsUmlLZFQrMmpDR0tNODBrMzA0Z0pFazFKMFloQVc0Q0xkeGRZbW9WSDJ4c2FWSFg0SkZWb1VXVmVQRk5rSTFJPSIsIml2UGFyYW1ldGVyU3BlYyI6InRQckQrWDdaY0l2RkZoWmUiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=develop)](https://us-east-1.console.aws.amazon.com/codesuite/codebuild/projects/automated-testing-oops-clang/history) +| Code Coverage | [![codecov](https://codecov.io/gh/JCSDA/oops/branch/develop/graph/badge.svg?token=GdDzbEQedm)](https://codecov.io/gh/JCSDA/oops) | diff --git a/cmake/backtrace_deps.cmake b/cmake/backtrace_deps.cmake new file mode 100644 index 000000000..e2d9cf8b7 --- /dev/null +++ b/cmake/backtrace_deps.cmake @@ -0,0 +1,64 @@ +# (C) Copyright 2021 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# This CMake file tests which stack trace provider libraries are available on +# a particular system. This is needed to set appropriate flags for +# boost stacktrace (used in src/oops/util/signal_trap.cc). + +include(CheckCXXSourceCompiles) + +# check_cxx_source_compiles uses global flags, unfortunately, so we need to +# save a bit of state to run the tests. + +set(saved_libraries ${CMAKE_REQUIRED_LIBRARIES}) +set(saved_defs ${CMAKE_REQUIRED_DEFINITIONS}) + +string(CONFIGURE [[ + #include + #include + + int main() { + std::cout << boost::stacktrace::stacktrace() << std::endl; + return 0; + } +]] stacktracecode @ONLY) + + +# Different configs + +list( APPEND OOPS_STACKTRACE_none_LIBS "") +set( OOPS_STACKTRACE_none_DEFS -DBOOST_STACKTRACE_USE_NOOP) + +list( APPEND OOPS_STACKTRACE_default_LIBS -ldl) +set( OOPS_STACKTRACE_default_DEFS "") + +list( APPEND OOPS_STACKTRACE_libbacktrace_LIBS -ldl -lbacktrace) +list( APPEND OOPS_STACKTRACE_libbacktrace_DEFS -DBOOST_STACKTRACE_USE_BACKTRACE) + +list( APPEND OOPS_STACKTRACE_addr2line_LIBS -ldl -lbacktrace) +list( APPEND OOPS_STACKTRACE_addr2line_DEFS -DBOOST_STACKTRACE_USE_ADDR2LINE) +find_program(addr2line_PATH addr2line) +if(addr2line_PATH) + message( STATUS "Found addr2line at ${addr2line_PATH}." ) + list(APPEND OOPS_STACKTRACE_addr2line_DEFS -DBOOST_STACKTRACE_ADDR2LINE_LOCATION=${addr2line_PATH}) +endif() + + +# Test each configuration here. +foreach ( provider IN ITEMS libbacktrace addr2line default none ) + list(APPEND CMAKE_REQUIRED_LIBRARIES ${OOPS_STACKTRACE_${provider}_LIBS}) + list(APPEND CMAKE_REQUIRED_DEFINITIONS ${OOPS_STACKTRACE_${provider}_DEFS}) + check_cxx_source_compiles("${stacktracecode}" OOPS_STACKTRACE_${provider}_AVAILABLE) + set(CMAKE_REQUIRED_LIBRARIES ${saved_libraries}) + set(CMAKE_REQUIRED_DEFINITIONS ${saved_defs}) + if ( OOPS_STACKTRACE_${provider}_AVAILABLE ) + list( APPEND OOPS_STACKTRACE_AVAILABLE_PROVIDERS ${provider} ) + endif() +endforeach() + +message( STATUS "Boost stacktrace supports these providers: ${OOPS_STACKTRACE_AVAILABLE_PROVIDERS}.") +list(GET OOPS_STACKTRACE_AVAILABLE_PROVIDERS 0 OOPS_STACKTRACE_PROVIDER) +message( STATUS "Using this provider for stacktraces: ${OOPS_STACKTRACE_PROVIDER}.") + diff --git a/cmake/oops_compiler_flags.cmake b/cmake/oops_compiler_flags.cmake index 383f91529..a8574563e 100644 --- a/cmake/oops_compiler_flags.cmake +++ b/cmake/oops_compiler_flags.cmake @@ -49,3 +49,5 @@ elseif( CMAKE_CXX_COMPILER_ID MATCHES "Clang" ) else() message( STATUS "C++ compiler with ID ${CMAKE_CXX_COMPILER_ID} will be used with CMake default options") endif() + + diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index e92a1a480..e7cd6f562 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -1127,13 +1127,6 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored diff --git a/docs/mainpage.h b/docs/mainpage.h new file mode 100644 index 000000000..2eb0772eb --- /dev/null +++ b/docs/mainpage.h @@ -0,0 +1,11 @@ +#pragma once + +// This file defines what appears on the Main Page of the documentation +// generated by doxygen. The file contains no code, and does not appear +// in any cpp include statement. +// +/*! + * \mainpage Object Oriented Prediction System (OOPS) + * + * OOPS is the top level of JEDI that orchestrates the configuration and execution of applications. This is where DA applications are implemented. OOPS also includes the Lorenz 95 and Quasi-geostrophic toy models that can used to run many of the DA applications. + */ diff --git a/ewok/4dvar.yaml b/ewok/4dvar.yaml index a0171f3df..0dd3a08cf 100644 --- a/ewok/4dvar.yaml +++ b/ewok/4dvar.yaml @@ -1,36 +1,36 @@ cost function: - cost type: $(cost_function) + cost type: ${cost_function} window begin: '{{window_begin}}' - window length: $(window_length) + window length: ${window_length} geometry: - $(GEOMETRY) + ${GEOMETRY} model: - $(MODEL) - analysis variables: $(an_variables) + ${MODEL} + analysis variables: ${an_variables} background: - $(BACKGROUND) + ${BACKGROUND} background error: - $(BACKGROUND_ERROR) + ${BACKGROUND_ERROR} observations: - $(OBSERVATIONS) + ${OBSERVATIONS} constraints: - - $(JC) + - ${JC} variational: minimizer: - $(MINIMIZER) + ${MINIMIZER} iterations: - - ninner: $(ninner) - gradient norm reduction: $(reduc) + - ninner: ${ninner} + gradient norm reduction: ${reduc} geometry: - $(GEOMETRY) + ${GEOMETRY} linear model: - $(LINEAR_MODEL) + ${LINEAR_MODEL} diagnostics: - departures: $(diag) + departures: ${diag} final: diagnostics: departures: oman prints: frequency: PT3H output: - $(AN_OUTPUT) + ${AN_OUTPUT} diff --git a/ewok/forecast.yaml b/ewok/forecast.yaml index ca7450e7b..2db88b23b 100644 --- a/ewok/forecast.yaml +++ b/ewok/forecast.yaml @@ -1,9 +1,9 @@ -forecast length: $(forecast_length) +forecast length: ${forecast_length} initial condition: - $(FC_INPUT) + ${FC_INPUT} geometry: - $(GEOMETRY) + ${GEOMETRY} model: - $(MODEL) + ${MODEL} output: - $(FC_OUTPUT) + ${FC_OUTPUT} diff --git a/ewok/hofx3d.yaml b/ewok/hofx3d.yaml index 824a56a45..379d269b0 100644 --- a/ewok/hofx3d.yaml +++ b/ewok/hofx3d.yaml @@ -1,9 +1,9 @@ geometry: - $(GEOMETRY) + ${GEOMETRY} state: '{{BACKGROUND}}' observations: - $(OBSERVATIONS) + ${OBSERVATIONS} window begin: '{{window_begin}}' -window length: $(window_length) +window length: ${window_length} diff --git a/ewok/hofx4d.yaml b/ewok/hofx4d.yaml index d688f3f92..0c09c3cd9 100644 --- a/ewok/hofx4d.yaml +++ b/ewok/hofx4d.yaml @@ -1,9 +1,9 @@ geometry: - $(GEOMETRY) + ${GEOMETRY} states: '{{BACKGROUND}}' observations: - $(OBSERVATIONS) + ${OBSERVATIONS} window begin: '{{window_begin}}' -window length: $(window_length) +window length: ${window_length} diff --git a/l95/ewok/README.md b/l95/ewok/README similarity index 100% rename from l95/ewok/README.md rename to l95/ewok/README diff --git a/l95/ewok/defaults/an.yaml b/l95/ewok/defaults/an.yaml index 34895a157..13d958337 100644 --- a/l95/ewok/defaults/an.yaml +++ b/l95/ewok/defaults/an.yaml @@ -1,2 +1,2 @@ date: '{{current_cycle}}' -filename: '$(current_dir)/$(experiment).an.{{current_cycle}}' +filename: '$(run_dir)/$(experiment).an.{{current_cycle}}' diff --git a/l95/ewok/defaults/an_output.yaml b/l95/ewok/defaults/an_output.yaml index 022f9c00d..5b7a6beee 100644 --- a/l95/ewok/defaults/an_output.yaml +++ b/l95/ewok/defaults/an_output.yaml @@ -1,5 +1,5 @@ -datadir: '$(current_dir)' -filename: '$(current_dir)/$(experiment).an.{{current_cycle}}' +datadir: $(run_dir) +filename: '$(run_dir)/$(experiment).an.{{current_cycle}}' exp: $(experiment) ##first: '{% $(window_length) - $(window_offset) %}' first: PT3H diff --git a/l95/ewok/defaults/bg.yaml b/l95/ewok/defaults/bg.yaml index 021233df2..c31d04b92 100644 --- a/l95/ewok/defaults/bg.yaml +++ b/l95/ewok/defaults/bg.yaml @@ -1,2 +1,2 @@ date: '{{window_begin}}' -filename: '$(current_dir)/$(experiment).bg.{{current_cycle}}.l95' +filename: '$(run_dir)/$(experiment).bg.{{current_cycle}}.l95' diff --git a/l95/ewok/defaults/fc.yaml b/l95/ewok/defaults/fc.yaml index c29dd966e..2db684737 100644 --- a/l95/ewok/defaults/fc.yaml +++ b/l95/ewok/defaults/fc.yaml @@ -1,2 +1,2 @@ date: '{{current_cycle}}' -filename: $(current_dir)/$(experiment).fc.{{current_cycle}}.$(step) +filename: $(run_dir)/$(experiment).fc.{{current_cycle}}.$(step) diff --git a/l95/ewok/defaults/fc_output.yaml b/l95/ewok/defaults/fc_output.yaml index b958e5abe..64f69a6e3 100644 --- a/l95/ewok/defaults/fc_output.yaml +++ b/l95/ewok/defaults/fc_output.yaml @@ -1,4 +1,4 @@ -datadir: $(current_dir) +datadir: $(run_dir) date: '{{current_cycle}}' exp: $(experiment) frequency: PT3H diff --git a/l95/ewok/defaults/obs.yaml b/l95/ewok/defaults/obs.yaml index 5df78463e..54d7977cb 100644 --- a/l95/ewok/defaults/obs.yaml +++ b/l95/ewok/defaults/obs.yaml @@ -3,8 +3,8 @@ obs operator: obs space: source: truth filetype: obt - obsdatain: '$(current_dir)/input.obs.{{window_begin}}.obt' - obsdataout: '$(current_dir)/$(experiment).obs.{{window_begin}}.obt' + obsdatain: '$(run_dir)/input.obs.{{window_begin}}.obt' + obsdataout: '$(run_dir)/$(experiment).obs.{{window_begin}}.obt' obs error: covariance model: diagonal r2d2_type: l95_obs diff --git a/l95/src/executables/EDA.cc b/l95/src/executables/EDA.cc index a4a3e9815..f72eaada8 100644 --- a/l95/src/executables/EDA.cc +++ b/l95/src/executables/EDA.cc @@ -5,6 +5,7 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/instantiateLocalizationFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/EnsembleApplication.h" @@ -13,6 +14,7 @@ int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); lorenz95::instantiateLocalizationFactory(); oops::EnsembleApplication >eda; return run.execute(eda); diff --git a/l95/src/executables/EnsHofX.cc b/l95/src/executables/EnsHofX.cc index 9a4a0f3be..8ff4f286c 100644 --- a/l95/src/executables/EnsHofX.cc +++ b/l95/src/executables/EnsHofX.cc @@ -8,6 +8,7 @@ * does it submit to any jurisdiction. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/EnsembleApplication.h" #include "oops/runs/HofX4D.h" @@ -15,6 +16,7 @@ int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); oops::EnsembleApplication > enshofx; return run.execute(enshofx); } diff --git a/l95/src/executables/HofX.cc b/l95/src/executables/HofX.cc index 5f9bfb119..75a20acef 100644 --- a/l95/src/executables/HofX.cc +++ b/l95/src/executables/HofX.cc @@ -8,12 +8,14 @@ * does it submit to any jurisdiction. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); oops::HofX4D hofx; return run.execute(hofx); } diff --git a/l95/src/executables/LETKF.cc b/l95/src/executables/LETKF.cc index fdea50c68..8d72e4d15 100644 --- a/l95/src/executables/LETKF.cc +++ b/l95/src/executables/LETKF.cc @@ -5,12 +5,14 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/LocalEnsembleDA.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); oops::LocalEnsembleDA letkf; return run.execute(letkf); } diff --git a/l95/src/executables/Main4dvar.cc b/l95/src/executables/Main4dvar.cc index 0ad6eab58..3f8dfe519 100644 --- a/l95/src/executables/Main4dvar.cc +++ b/l95/src/executables/Main4dvar.cc @@ -8,6 +8,7 @@ * does it submit to any jurisdiction. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/instantiateLocalizationFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/Run.h" @@ -15,6 +16,7 @@ int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); lorenz95::instantiateLocalizationFactory(); oops::Variational var; return run.execute(var); diff --git a/l95/src/lorenz95/BackgroundCheck.cc b/l95/src/lorenz95/BackgroundCheck.cc index 1705d257a..a7989398b 100644 --- a/l95/src/lorenz95/BackgroundCheck.cc +++ b/l95/src/lorenz95/BackgroundCheck.cc @@ -10,13 +10,12 @@ #include #include "lorenz95/L95Traits.h" -#include "oops/interface/ObsFilter.h" // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -static oops::FilterMaker > makerBackgroundCheck_("Background Check"); +static oops::interface::FilterMaker makerBackgroundCheck_( + "Background Check"); // ----------------------------------------------------------------------------- BackgroundCheck::BackgroundCheck(const ObsTable & obsdb, const Parameters_ & parameters, @@ -26,7 +25,7 @@ BackgroundCheck::BackgroundCheck(const ObsTable & obsdb, const Parameters_ & par } // ----------------------------------------------------------------------------- -void BackgroundCheck::postFilter(const ObsVec1D & hofx, const ObsDiags1D &) const { +void BackgroundCheck::postFilter(const ObsVec1D & hofx, const ObsVec1D &, const ObsDiags1D &) { std::vector yobs; obsdb_.getdb("ObsValue", yobs); size_t inflate = 0; diff --git a/l95/src/lorenz95/BackgroundCheck.h b/l95/src/lorenz95/BackgroundCheck.h index 5c05b80a6..ec11f9f5a 100644 --- a/l95/src/lorenz95/BackgroundCheck.h +++ b/l95/src/lorenz95/BackgroundCheck.h @@ -11,11 +11,13 @@ #include #include -#include "oops/base/ObsFilterParametersBase.h" +#include "lorenz95/L95Traits.h" + #include "oops/base/Variables.h" +#include "oops/generic/ObsFilterParametersBase.h" +#include "oops/interface/ObsFilterBase.h" #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/RequiredParameter.h" -#include "oops/util/Printable.h" namespace lorenz95 { class GomL95; @@ -40,22 +42,22 @@ class BackgroundCheckParameters : public oops::ObsFilterParametersBase { }; /// Simple background check: all obs for which {|y-H(x)| < threshold} pass QC -class BackgroundCheck : public util::Printable { +class BackgroundCheck : public oops::interface::ObsFilterBase { public: typedef BackgroundCheckParameters Parameters_; BackgroundCheck(const ObsTable &, const Parameters_ &, std::shared_ptr >, std::shared_ptr >); - void preProcess() const {} - void priorFilter(const GomL95 &) const {} - void postFilter(const ObsVec1D &, const ObsDiags1D &) const; + void preProcess() override {} + void priorFilter(const GomL95 &) override {} + void postFilter(const ObsVec1D &, const ObsVec1D &, const ObsDiags1D &) override; - oops::Variables requiredVars() const {return novars_;} - oops::Variables requiredHdiagnostics() const {return novars_;} + oops::Variables requiredVars() const override {return novars_;} + oops::Variables requiredHdiagnostics() const override {return novars_;} private: - void print(std::ostream & os) const; + void print(std::ostream & os) const override; const ObsTable & obsdb_; Parameters_ options_; diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index 73bfb56ce..2db1f2cb3 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -7,6 +7,8 @@ set( _l95_srcs ErrorCovarianceL95.h FieldL95.cc FieldL95.h + FinalCheck.cc + FinalCheck.h GetValuesL95.cc GetValuesL95.h GetValuesTLAD.cc diff --git a/l95/src/lorenz95/FinalCheck.cc b/l95/src/lorenz95/FinalCheck.cc new file mode 100644 index 000000000..435319b0b --- /dev/null +++ b/l95/src/lorenz95/FinalCheck.cc @@ -0,0 +1,20 @@ +/* + * (C) Crown Copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/FinalCheck.h" + +#include + +#include "lorenz95/L95Traits.h" + +// ----------------------------------------------------------------------------- +namespace lorenz95 { +// ----------------------------------------------------------------------------- +static oops::interface::FilterMaker makerQCm_("Final Check"); +// ----------------------------------------------------------------------------- +} // namespace lorenz95 + diff --git a/l95/src/lorenz95/FinalCheck.h b/l95/src/lorenz95/FinalCheck.h new file mode 100644 index 000000000..d369535b9 --- /dev/null +++ b/l95/src/lorenz95/FinalCheck.h @@ -0,0 +1,49 @@ +/* + * (C) Crown Copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef LORENZ95_FINALCHECK_H_ +#define LORENZ95_FINALCHECK_H_ + +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "lorenz95/L95Traits.h" + +#include "oops/base/Variables.h" +#include "oops/interface/ObsFilterBase.h" + +namespace lorenz95 { + class GomL95; + template class ObsData1D; + class ObsTable; + class ObsDiags1D; + class ObsVec1D; + +// Nothing to do here for the Lorenz model + +class FinalCheck : public oops::interface::ObsFilterBase { + public: + FinalCheck(const ObsTable &, const eckit::Configuration &, + std::shared_ptr >, std::shared_ptr >): novars_() {} + + void preProcess() override {} + void priorFilter(const GomL95 &) override {} + void postFilter(const ObsVec1D &, const ObsVec1D &, const ObsDiags1D &) override {} + + oops::Variables requiredVars() const override {return novars_;} + oops::Variables requiredHdiagnostics() const override {return novars_;} + + private: + void print(std::ostream &) const override {} + const oops::Variables novars_; +}; + +} // namespace lorenz95 + +#endif // LORENZ95_FINALCHECK_H_ diff --git a/l95/src/lorenz95/GomL95.cc b/l95/src/lorenz95/GomL95.cc index 95a7cb621..0dc74c38d 100644 --- a/l95/src/lorenz95/GomL95.cc +++ b/l95/src/lorenz95/GomL95.cc @@ -30,7 +30,7 @@ class Variables; namespace lorenz95 { // ----------------------------------------------------------------------------- -GomL95::GomL95(const LocsL95 & locs, const oops::Variables &) +GomL95::GomL95(const LocsL95 & locs, const oops::Variables &, const std::vector &) : size_(locs.size()), locval_(size_) { oops::Log::trace() << "GomL95::GomL95 starting " << std::endl; diff --git a/l95/src/lorenz95/GomL95.h b/l95/src/lorenz95/GomL95.h index 058298322..abf669490 100644 --- a/l95/src/lorenz95/GomL95.h +++ b/l95/src/lorenz95/GomL95.h @@ -34,7 +34,7 @@ class GomL95 : public util::Printable, public: static const std::string classname() {return "lorenz95::GomL95";} - GomL95(const LocsL95 &, const oops::Variables &); + GomL95(const LocsL95 &, const oops::Variables &, const std::vector &); GomL95(const eckit::Configuration &, const ObsTable &, const oops::Variables &); diff --git a/l95/src/lorenz95/LocalizationMatrixL95.h b/l95/src/lorenz95/LocalizationMatrixL95.h index 0e29b43b4..af099076a 100644 --- a/l95/src/lorenz95/LocalizationMatrixL95.h +++ b/l95/src/lorenz95/LocalizationMatrixL95.h @@ -18,9 +18,8 @@ #include #include "eckit/config/Configuration.h" +#include "oops/interface/LocalizationBase.h" #include "oops/util/DateTime.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" #include "lorenz95/L95Traits.h" @@ -31,14 +30,13 @@ namespace lorenz95 { /// Localization matrix for Lorenz 95 model. // ----------------------------------------------------------------------------- -class LocalizationMatrixL95: public util::Printable, - private util::ObjectCounter { +class LocalizationMatrixL95: public oops::interface::LocalizationBase { public: static const std::string classname() {return "lorenz95::LocalizationMatrixL95";} LocalizationMatrixL95(const Resolution &, const eckit::Configuration &); - void randomize(IncrementL95 &) const; - void multiply(IncrementL95 &) const; + void randomize(IncrementL95 &) const override; + void multiply(IncrementL95 &) const override; private: void print(std::ostream &) const override; diff --git a/l95/src/lorenz95/ModelBias.cc b/l95/src/lorenz95/ModelBias.cc index d526b8b3a..da1a3b8fb 100644 --- a/l95/src/lorenz95/ModelBias.cc +++ b/l95/src/lorenz95/ModelBias.cc @@ -48,7 +48,7 @@ ModelBias & ModelBias::operator+=(const ModelBiasCorrection & dx) { } // ----------------------------------------------------------------------------- void ModelBias::print(std::ostream & os) const { - if (active_) {os << std::endl << "ModelBias = " << bias_;} + if (active_) {os << "ModelBias = " << bias_;} } // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/src/lorenz95/ModelL95.cc b/l95/src/lorenz95/ModelL95.cc index 8215a3222..5d1be6796 100644 --- a/l95/src/lorenz95/ModelL95.cc +++ b/l95/src/lorenz95/ModelL95.cc @@ -24,7 +24,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -static oops::ModelMaker makermodel_("L95"); +static oops::interface::ModelMaker makermodel_("L95"); // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ModelL95.h b/l95/src/lorenz95/ModelL95.h index 61c326835..0cae84980 100644 --- a/l95/src/lorenz95/ModelL95.h +++ b/l95/src/lorenz95/ModelL95.h @@ -15,8 +15,8 @@ #include #include "eckit/config/Configuration.h" -#include "oops/base/ModelBase.h" #include "oops/base/Variables.h" +#include "oops/interface/ModelBase.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/ObjectCounter.h" @@ -47,7 +47,7 @@ class ModelL95Parameters : public oops::ModelParametersBase { // ----------------------------------------------------------------------------- -class ModelL95 : public oops::ModelBase, +class ModelL95 : public oops::interface::ModelBase, private util::ObjectCounter { public: typedef ModelL95Parameters Parameters_; diff --git a/l95/src/lorenz95/ObsBias.cc b/l95/src/lorenz95/ObsBias.cc index 95a7e306e..3f754db1d 100644 --- a/l95/src/lorenz95/ObsBias.cc +++ b/l95/src/lorenz95/ObsBias.cc @@ -49,7 +49,7 @@ ObsBias & ObsBias::operator=(const ObsBias & rhs) { } // ----------------------------------------------------------------------------- void ObsBias::print(std::ostream & os) const { - if (active_) {os << std::endl << "ObsBias = " << bias_;} + if (active_) {os << "ObsBias = " << bias_;} } // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/src/lorenz95/ObsData1D.h b/l95/src/lorenz95/ObsData1D.h index 316de71fa..4453121bd 100644 --- a/l95/src/lorenz95/ObsData1D.h +++ b/l95/src/lorenz95/ObsData1D.h @@ -80,8 +80,8 @@ ObsData1D::ObsData1D(const ObsData1D & other) template ObsData1D::ObsData1D(const ObsVec1D & other) : obsdb_(other.obsdb()), data_(other.size()) { - const DATATYPE missing = util::missingValue(missing); - const double dmiss = util::missingValue(dmiss); + const DATATYPE missing = util::missingValue(DATATYPE()); + const double dmiss = util::missingValue(double()); for (size_t jj = 0; jj < data_.size(); ++jj) { if (other[jj] == dmiss) { data_.at(jj) = missing; @@ -107,7 +107,7 @@ void ObsData1D::zero() { // ----------------------------------------------------------------------------- template void ObsData1D::mask(const ObsData1D & mask) { - DATATYPE missing = util::missingValue(missing); + DATATYPE missing = util::missingValue(DATATYPE()); for (size_t jj = 0; jj < data_.size(); ++jj) { if (mask[jj]) data_.at(jj) = missing; } @@ -125,7 +125,7 @@ void ObsData1D::save(const std::string & name) const { // ----------------------------------------------------------------------------- template void ObsData1D::print(std::ostream & os) const { - DATATYPE missing = util::missingValue(missing); + DATATYPE missing = util::missingValue(DATATYPE()); DATATYPE zmin = std::numeric_limits::max(); DATATYPE zmax = std::numeric_limits::lowest(); DATATYPE zavg = 0.0; diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index 2a3cf6673..ffe9b365c 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -35,18 +35,16 @@ ObsLocGC99::ObsLocGC99(const eckit::Configuration & config, const ObsTable & obs // ----------------------------------------------------------------------------- -void ObsLocGC99::computeLocalization(const Iterator & iterator, ObsData1D & outside, - ObsVec1D & result) const { +void ObsLocGC99::computeLocalization(const Iterator & iterator, ObsVec1D & locfactor) const { std::vector locations = obsdb_.locations(); eckit::geometry::Point2 center = *iterator; for (unsigned int ii=0; ii < obsdb_.nobs(); ++ii) { double curdist = std::abs(center[0] - locations[ii]); curdist = std::min(curdist, 1.-curdist); if (curdist >= rscale_) { - outside[ii] = 1; + locfactor[ii] = locfactor.missing(); } else { - outside[ii] = 0; - result[ii] = oops::gc99(curdist/rscale_); + locfactor[ii] = oops::gc99(curdist/rscale_); } } } diff --git a/l95/src/lorenz95/ObsLocGC99.h b/l95/src/lorenz95/ObsLocGC99.h index c7695cb6b..a7607a99e 100644 --- a/l95/src/lorenz95/ObsLocGC99.h +++ b/l95/src/lorenz95/ObsLocGC99.h @@ -14,7 +14,6 @@ #include "oops/base/ObsLocalizationBase.h" #include "lorenz95/L95Traits.h" -#include "lorenz95/ObsData1D.h" // Forward declarations namespace lorenz95 { @@ -27,11 +26,9 @@ class ObsLocGC99: public oops::ObsLocalizationBase { public: ObsLocGC99(const eckit::Configuration &, const ObsTable &); - /// compute localization and save localization values in \p obsvector and - /// localization flags (1: outside of localization; 0: inside localization area) - /// in \p outside - void computeLocalization(const Iterator &, ObsData1D & outside, - ObsVec1D & obsvector) const override; + /// compute localization and save localization values in \p locfactor + /// (missing value is for obs outside of localization) + void computeLocalization(const Iterator &, ObsVec1D & locfactor) const override; private: void print(std::ostream &) const override; diff --git a/l95/src/lorenz95/ObsTable.cc b/l95/src/lorenz95/ObsTable.cc index 0852fe98f..3faa36b92 100644 --- a/l95/src/lorenz95/ObsTable.cc +++ b/l95/src/lorenz95/ObsTable.cc @@ -39,27 +39,24 @@ namespace sf = util::stringfunctions; namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsTable::ObsTable(const eckit::Configuration & config, const eckit::mpi::Comm & comm, +ObsTable::ObsTable(const Parameters_ & params, const eckit::mpi::Comm & comm, const util::DateTime & bgn, const util::DateTime & end, const eckit::mpi::Comm & timeComm) - : oops::ObsSpaceBase(config, comm, bgn, end), winbgn_(bgn), winend_(end), comm_(timeComm), + : oops::ObsSpaceBase(params, comm, bgn, end), winbgn_(bgn), winend_(end), comm_(timeComm), obsvars_() { oops::Log::trace() << "ObsTable::ObsTable starting" << std::endl; - nameIn_.clear(); - nameOut_.clear(); - if (config.has("obsdatain")) { - nameIn_ = config.getString("obsdatain"); + if (params.obsdatain.value() != boost::none) { + nameIn_ = *params.obsdatain.value(); otOpen(nameIn_); } // Generate locations etc... if required - if (config.has("generate")) { - const eckit::LocalConfiguration gconf(config, "generate"); - generateDistribution(gconf); + if (params.generate.value() != boost::none) { + generateDistribution(*params.generate.value()); } - if (config.has("obsdataout")) { - nameOut_ = config.getString("obsdataout"); - sf::swapNameMember(config, nameOut_); + if (params.obsdataout.value() != boost::none) { + nameOut_ = *params.obsdataout.value(); + sf::swapNameMember(params.toConfiguration(), nameOut_); } oops::Log::trace() << "ObsTable::ObsTable created nobs = " << nobs() << std::endl; } @@ -86,8 +83,8 @@ bool ObsTable::has(const std::string & col) const { void ObsTable::putdb(const std::string & col, const std::vector & vec) const { std::vector tmp(vec.size()); - int intmiss = util::missingValue(intmiss); - double doublemiss = util::missingValue(doublemiss); + int intmiss = util::missingValue(int()); + double doublemiss = util::missingValue(double()); for (size_t jobs = 0; jobs < vec.size(); ++jobs) { if (vec[jobs] == intmiss) { tmp[jobs] = doublemiss; @@ -102,8 +99,8 @@ void ObsTable::putdb(const std::string & col, const std::vector & vec) cons void ObsTable::putdb(const std::string & col, const std::vector & vec) const { std::vector tmp(vec.size()); - float floatmiss = util::missingValue(floatmiss); - double doublemiss = util::missingValue(doublemiss); + float floatmiss = util::missingValue(float()); + double doublemiss = util::missingValue(double()); for (size_t jobs = 0; jobs < vec.size(); ++jobs) { if (vec[jobs] == floatmiss) { tmp[jobs] = doublemiss; @@ -131,8 +128,8 @@ void ObsTable::putdb(const std::string & col, const std::vector & vec) c void ObsTable::getdb(const std::string & col, std::vector & vec) const { std::vector tmp; this->getdb(col, tmp); - int intmiss = util::missingValue(intmiss); - double doublemiss = util::missingValue(doublemiss); + int intmiss = util::missingValue(int()); + double doublemiss = util::missingValue(double()); vec.resize(nobs()); for (size_t jobs = 0; jobs < nobs(); ++jobs) { if (tmp[jobs] == doublemiss) { @@ -148,8 +145,8 @@ void ObsTable::getdb(const std::string & col, std::vector & vec) const { void ObsTable::getdb(const std::string & col, std::vector & vec) const { std::vector tmp; this->getdb(col, tmp); - float floatmiss = util::missingValue(floatmiss); - double doublemiss = util::missingValue(doublemiss); + float floatmiss = util::missingValue(float()); + double doublemiss = util::missingValue(double()); vec.resize(nobs()); for (size_t jobs = 0; jobs < nobs(); ++jobs) { if (tmp[jobs] == doublemiss) { @@ -176,15 +173,15 @@ void ObsTable::getdb(const std::string & col, std::vector & vec) const { // ----------------------------------------------------------------------------- -void ObsTable::generateDistribution(const eckit::Configuration & config) { +void ObsTable::generateDistribution(const ObsGenerateParameters & params) { oops::Log::trace() << "ObsTable::generateDistribution starting" << std::endl; - util::Duration first(config.getString("begin")); + const util::Duration &first = params.begin; util::Duration last(winend_-winbgn_); - if (config.has("end")) { - last = util::Duration(config.getString("end")); + if (params.end.value() != boost::none) { + last = *params.end.value(); } - util::Duration freq(config.getString("obs_frequency")); + const util::Duration &freq = params.obsFrequency; int nobstimes = 0; util::Duration step(first); @@ -193,7 +190,7 @@ void ObsTable::generateDistribution(const eckit::Configuration & config) { step += freq; } - const unsigned int nobs_locations = config.getInt("obs_density"); + const unsigned int nobs_locations = params.obsDensity; const unsigned int nobs = nobs_locations*nobstimes; double dx = 1.0/static_cast(nobs_locations); @@ -216,7 +213,7 @@ void ObsTable::generateDistribution(const eckit::Configuration & config) { ASSERT(iobs == nobs); // Generate obs error - const double err = config.getDouble("obs_error"); + const double err = params.obsError; std::vector obserr(nobs); for (unsigned int jj = 0; jj < nobs; ++jj) { obserr[jj] = err; diff --git a/l95/src/lorenz95/ObsTable.h b/l95/src/lorenz95/ObsTable.h index ea20f8292..051e82d75 100644 --- a/l95/src/lorenz95/ObsTable.h +++ b/l95/src/lorenz95/ObsTable.h @@ -22,27 +22,55 @@ #include "oops/base/Variables.h" #include "oops/util/DateTime.h" #include "oops/util/ObjectCounter.h" - -namespace eckit { - class Configuration; -} +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" namespace lorenz95 { class ObsIterator; +// ----------------------------------------------------------------------------- +/// Options controlling generation of artificial observations. +class ObsGenerateParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGenerateParameters, Parameters) + + public: + oops::RequiredParameter begin{"begin", this}; + oops::OptionalParameter end{"end", this}; + oops::RequiredParameter obsFrequency{"obs_frequency", this}; + /// Number of observations to generate in each time slot. + oops::RequiredParameter obsDensity{"obs_density", this}; + oops::RequiredParameter obsError{"obs_error", this}; +}; + +// ----------------------------------------------------------------------------- +/// \brief Configuration parameters for the L95 model's ObsSpace. +class ObsTableParameters : public oops::ObsSpaceParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsTableParameters, ObsSpaceParametersBase) + + public: + /// File from which to load observations. + oops::OptionalParameter obsdatain{"obsdatain", this}; + /// File to which to save observations and analysis. + oops::OptionalParameter obsdataout{"obsdataout", this}; + /// Options controlling generation of artificial observations. + oops::OptionalParameter generate{"generate", this}; +}; + +// ----------------------------------------------------------------------------- /// A Simple Observation Data Handler /*! * ObsTable defines a simple observation handler * that mimicks the interfaces required from ODB. */ - -// ----------------------------------------------------------------------------- class ObsTable : public oops::ObsSpaceBase, private util::ObjectCounter { public: static const std::string classname() {return "lorenz95::ObsTable";} - ObsTable(const eckit::Configuration &, const eckit::mpi::Comm &, + typedef ObsTableParameters Parameters_; + + ObsTable(const Parameters_ &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); ~ObsTable(); @@ -56,7 +84,7 @@ class ObsTable : public oops::ObsSpaceBase, void getdb(const std::string &, std::vector &) const; bool has(const std::string & col) const; - void generateDistribution(const eckit::Configuration &); + void generateDistribution(const ObsGenerateParameters & params); void random(std::vector &) const; unsigned int nobs() const {return times_.size();} const std::vector & locations() const { return locations_; } diff --git a/l95/src/lorenz95/ObsVec1D.cc b/l95/src/lorenz95/ObsVec1D.cc index 76446d2f0..c729c5248 100644 --- a/l95/src/lorenz95/ObsVec1D.cc +++ b/l95/src/lorenz95/ObsVec1D.cc @@ -162,8 +162,14 @@ void ObsVec1D::mask(const ObsData1D & mask) { } } // ----------------------------------------------------------------------------- +void ObsVec1D::mask(const ObsVec1D & mask) { + for (size_t jj = 0; jj < data_.size(); ++jj) { + if (mask[jj] == missing_) data_.at(jj) = missing_; + } +} +// ----------------------------------------------------------------------------- ObsVec1D & ObsVec1D::operator=(const ObsData1D & rhs) { - const float fmiss = util::missingValue(fmiss); + const float fmiss = util::missingValue(float()); for (size_t jj = 0; jj < data_.size(); ++jj) { if (rhs[jj] == fmiss) { data_.at(jj) = missing_; @@ -178,21 +184,21 @@ void ObsVec1D::save(const std::string & name) const { obsdb_.putdb(name, data_); } // ----------------------------------------------------------------------------- -Eigen::VectorXd ObsVec1D::packEigen(const ObsData1D & mask) const { +Eigen::VectorXd ObsVec1D::packEigen(const ObsVec1D & mask) const { Eigen::VectorXd vec(packEigenSize(mask)); size_t ii = 0; for (size_t jj = 0; jj < data_.size(); ++jj) { - if ((data_[jj] != missing_) && (mask[jj] == 0)) { + if ((data_[jj] != missing_) && (mask[jj] != missing_)) { vec(ii++) = data_[jj]; } } return vec; } // ----------------------------------------------------------------------------- -size_t ObsVec1D::packEigenSize(const ObsData1D & mask) const { +size_t ObsVec1D::packEigenSize(const ObsVec1D & mask) const { size_t ii = 0; for (size_t jj = 0; jj < data_.size(); ++jj) { - if ((data_[jj] != missing_) && (mask[jj] == 0)) { + if ((data_[jj] != missing_) && (mask[jj] != missing_)) { ii++; } } diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index c9c3cbae9..854d8fd0c 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -45,8 +45,8 @@ class ObsVec1D : public util::Printable, ObsVec1D & operator*= (const ObsVec1D &); ObsVec1D & operator/= (const ObsVec1D &); - Eigen::VectorXd packEigen(const ObsData1D &) const; - size_t packEigenSize(const ObsData1D &) const; + Eigen::VectorXd packEigen(const ObsVec1D &) const; + size_t packEigenSize(const ObsVec1D &) const; size_t size() const {return data_.size();} const double & operator[](const std::size_t ii) const {return data_.at(ii);} @@ -62,6 +62,7 @@ class ObsVec1D : public util::Printable, double dot_product_with(const ObsVec1D &) const; double rms() const; void mask(const ObsData1D &); + void mask(const ObsVec1D &); ObsVec1D & operator= (const ObsData1D &); unsigned int nobs() const; @@ -71,6 +72,8 @@ class ObsVec1D : public util::Printable, void save(const std::string &) const; void read(const std::string &); + const double & missing() const {return missing_;} + private: void print(std::ostream &) const; diff --git a/l95/src/lorenz95/ObservationL95.cc b/l95/src/lorenz95/ObservationL95.cc index 449a6d2d1..89ffd0630 100644 --- a/l95/src/lorenz95/ObservationL95.cc +++ b/l95/src/lorenz95/ObservationL95.cc @@ -26,7 +26,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -ObservationL95::ObservationL95(const ObsTable & ot, const eckit::Configuration &) +ObservationL95::ObservationL95(const ObsTable & ot, const Parameters_ &) : obsdb_(ot), inputs_(std::vector{"x"}) {} @@ -37,7 +37,7 @@ ObservationL95::~ObservationL95() {} // ----------------------------------------------------------------------------- void ObservationL95::simulateObs(const GomL95 & gom, ObsVec1D & ovec, - const ObsBias & bias, ObsDiags1D &) const { + const ObsBias & bias, ObsVec1D &, ObsDiags1D &) const { for (size_t jj = 0; jj < gom.size(); ++jj) { ovec[jj] = gom[jj] + bias.value(); } diff --git a/l95/src/lorenz95/ObservationL95.h b/l95/src/lorenz95/ObservationL95.h index f4757dde7..0bda67bc0 100644 --- a/l95/src/lorenz95/ObservationL95.h +++ b/l95/src/lorenz95/ObservationL95.h @@ -47,13 +47,15 @@ class ObservationL95 : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObservationL95Parameters Parameters_; + static const std::string classname() {return "lorenz95::ObservationL95";} - ObservationL95(const ObsTable &, const eckit::Configuration &); + ObservationL95(const ObsTable &, const Parameters_ &); ~ObservationL95(); // Obs Operators - void simulateObs(const GomL95 &, ObsVec1D &, const ObsBias &, ObsDiags1D &) const; + void simulateObs(const GomL95 &, ObsVec1D &, const ObsBias &, ObsVec1D &, ObsDiags1D &) const; // Other const oops::Variables & requiredVars() const {return inputs_;} diff --git a/l95/src/lorenz95/ObservationTLAD.cc b/l95/src/lorenz95/ObservationTLAD.cc index d0d235046..15f849a7a 100644 --- a/l95/src/lorenz95/ObservationTLAD.cc +++ b/l95/src/lorenz95/ObservationTLAD.cc @@ -24,7 +24,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -ObservationTLAD::ObservationTLAD(const ObsTable &, const eckit::Configuration &) +ObservationTLAD::ObservationTLAD(const ObsTable &, const Parameters_ &) : inputs_(std::vector{"x"}) {} @@ -45,7 +45,7 @@ void ObservationTLAD::simulateObsTL(const GomL95 & gom, ObsVec1D & ovec, void ObservationTLAD::simulateObsAD(GomL95 & gom, const ObsVec1D & ovec, ObsBiasCorrection & bias) const { - const double missing = util::missingValue(missing); + const double missing = util::missingValue(double()); for (size_t jj = 0; jj < gom.size(); ++jj) { if (ovec[jj] != missing) { gom[jj] = ovec[jj]; diff --git a/l95/src/lorenz95/ObservationTLAD.h b/l95/src/lorenz95/ObservationTLAD.h index c3ba0b72f..394606aae 100644 --- a/l95/src/lorenz95/ObservationTLAD.h +++ b/l95/src/lorenz95/ObservationTLAD.h @@ -20,6 +20,7 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/Parameters.h" #include "oops/util/Printable.h" // Forward declarations @@ -33,20 +34,30 @@ namespace lorenz95 { class ObsBiasCorrection; class ObsVec1D; +// ----------------------------------------------------------------------------- + +/// (Empty) parameters controlling the observation operator for the Lorenz 95 model. +class ObservationL95Parameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObservationL95Parameters, Parameters); + // No parameters needed. +}; + +// ----------------------------------------------------------------------------- + /// Observation for Lorenz 95 model. /*! * ObservationTLAD defines the linearized ObsOperator for Lorenz 95 model. */ -// ----------------------------------------------------------------------------- - class ObservationTLAD : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObservationL95Parameters Parameters_; + static const std::string classname() {return "lorenz95::ObservationTLAD";} - ObservationTLAD(const ObsTable &, const eckit::Configuration &); + ObservationTLAD(const ObsTable &, const Parameters_ &); // Obs Operators void setTrajectory(const GomL95 &, const ObsBias &); diff --git a/l95/src/lorenz95/QCmanager.cc b/l95/src/lorenz95/QCmanager.cc index df1e1deaf..84cfc788d 100644 --- a/l95/src/lorenz95/QCmanager.cc +++ b/l95/src/lorenz95/QCmanager.cc @@ -10,13 +10,11 @@ #include #include "lorenz95/L95Traits.h" -#include "oops/interface/ObsFilter.h" // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -static oops::FilterMaker > makerQCm_("QCmanager"); +static oops::interface::FilterMaker makerQCm_("QCmanager"); // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/src/lorenz95/QCmanager.h b/l95/src/lorenz95/QCmanager.h index 6fafa33b5..91ff810eb 100644 --- a/l95/src/lorenz95/QCmanager.h +++ b/l95/src/lorenz95/QCmanager.h @@ -13,8 +13,11 @@ #include "eckit/config/LocalConfiguration.h" +#include "lorenz95/L95Traits.h" + #include "oops/base/Variables.h" -#include "oops/util/Printable.h" +#include "oops/generic/ObsFilterParametersBase.h" +#include "oops/interface/ObsFilterBase.h" namespace lorenz95 { class GomL95; @@ -25,21 +28,21 @@ namespace lorenz95 { // Nothing to do here for the Lorenz model -class QCmanager : public util::Printable { +class QCmanager : public oops::interface::ObsFilterBase { public: QCmanager(const ObsTable &, const eckit::Configuration &, std::shared_ptr >, std::shared_ptr >): novars_() {} ~QCmanager() {} - void preProcess() const {} - void priorFilter(const GomL95 &) const {} - void postFilter(const ObsVec1D &, const ObsDiags1D &) const {} + void preProcess() override {} + void priorFilter(const GomL95 &) override {} + void postFilter(const ObsVec1D &, const ObsVec1D &, const ObsDiags1D &) override {} - oops::Variables requiredVars() const {return novars_;} - oops::Variables requiredHdiagnostics() const {return novars_;} + oops::Variables requiredVars() const override {return novars_;} + oops::Variables requiredHdiagnostics() const override {return novars_;} private: - void print(std::ostream &) const {} + void print(std::ostream &) const override {} const oops::Variables novars_; }; diff --git a/l95/src/lorenz95/Resolution.cc b/l95/src/lorenz95/Resolution.cc index d52280054..15bf6a6ab 100644 --- a/l95/src/lorenz95/Resolution.cc +++ b/l95/src/lorenz95/Resolution.cc @@ -27,6 +27,11 @@ std::vector Resolution::verticalCoord(std::string & vcUnits) const { std::vector vc(1, 1.0); return vc; } +// ------------------------------------------------------------------------------------------------- +std::vector Resolution::variableSizes(const oops::Variables & vars) const { + std::vector sizes(vars.size(), 1); + return sizes; +} // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/Resolution.h b/l95/src/lorenz95/Resolution.h index a4f51289d..587fbf442 100644 --- a/l95/src/lorenz95/Resolution.h +++ b/l95/src/lorenz95/Resolution.h @@ -17,6 +17,7 @@ #include "eckit/config/Configuration.h" #include "lorenz95/Iterator.h" +#include "oops/base/Variables.h" #include "oops/mpi/mpi.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" @@ -54,6 +55,7 @@ class Resolution : public util::Printable { Iterator begin() const; Iterator end() const; std::vector verticalCoord(std::string &) const; + std::vector variableSizes(const oops::Variables &) const; const eckit::mpi::Comm & getComm() const {return comm_;} private: diff --git a/l95/src/lorenz95/TLML95.cc b/l95/src/lorenz95/TLML95.cc index 5b66fbe23..dd533af88 100644 --- a/l95/src/lorenz95/TLML95.cc +++ b/l95/src/lorenz95/TLML95.cc @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -31,7 +31,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -static oops::LinearModelMaker makerTLML95_("L95TLM"); +static oops::interface::LinearModelMaker makerTLML95_("L95TLM"); // ----------------------------------------------------------------------------- TLML95::TLML95(const Resolution & resol, const Parameters_ & params) : resol_(resol), tstep_(params.tstep), diff --git a/l95/src/lorenz95/TLML95.h b/l95/src/lorenz95/TLML95.h index cc6cf37dd..18d8a6c2d 100644 --- a/l95/src/lorenz95/TLML95.h +++ b/l95/src/lorenz95/TLML95.h @@ -17,8 +17,8 @@ #include -#include "oops/base/LinearModelBase.h" #include "oops/base/Variables.h" +#include "oops/interface/LinearModelBase.h" #include "oops/util/Duration.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -55,7 +55,7 @@ class TLML95Parameters : public oops::LinearModelParametersBase { // ----------------------------------------------------------------------------- /// Lorenz 95 linear model definition. -class TLML95: public oops::LinearModelBase, +class TLML95: public oops::interface::LinearModelBase, private util::ObjectCounter { public: typedef TLML95Parameters Parameters_; diff --git a/l95/src/lorenz95/instantiateLocalizationFactory.h b/l95/src/lorenz95/instantiateLocalizationFactory.h index ff4753449..093d5c755 100644 --- a/l95/src/lorenz95/instantiateLocalizationFactory.h +++ b/l95/src/lorenz95/instantiateLocalizationFactory.h @@ -10,13 +10,12 @@ #include "lorenz95/L95Traits.h" #include "lorenz95/LocalizationMatrixL95.h" -#include "oops/interface/Localization.h" +#include "oops/interface/LocalizationBase.h" namespace lorenz95 { void instantiateLocalizationFactory() { - static oops::LocalizationMaker> makerL95_("L95"); + static oops::interface::LocalizationMaker makerL95_("L95"); } } // namespace lorenz95 diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index e883dfe72..4a062170a 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -20,6 +20,7 @@ list( APPEND l95_test_input testinput/4dvar.drpcgqn.yaml testinput/4dvar.drplanczos.yaml testinput/4dvar.drplanclmp.yaml + testinput/4dvar.drplzero.yaml testinput/4dvar.fgmres.yaml testinput/4dvar.gmresr.yaml testinput/4dvar.hybrid.yaml @@ -80,6 +81,7 @@ list( APPEND l95_test_input testinput/makeobsbias.yaml testinput/makeobspert.yaml testinput/pseudomodel.yaml + testinput/pseudomodel_state4d.yaml testinput/truth.yaml testinput/simplifiedl95_DRGMRESR.yaml testinput/simplifiedl95_DRIPCG.yaml @@ -128,6 +130,7 @@ list( APPEND l95_testoutput testoutput/4dvar.drpcgqn.test testoutput/4dvar.drplanclmp.test testoutput/4dvar.drplanczos.test + testoutput/4dvar.drplzero.test testoutput/4dvar.fgmres.test testoutput/4dvar.gmresr.test testoutput/4dvar.hybrid.test @@ -283,6 +286,12 @@ ecbuild_add_test( TARGET test_l95_pseudomodel LIBS lorenz95 TEST_DEPENDS test_l95_truth ) +ecbuild_add_test( TARGET test_l95_pseudomodel_state4d + SOURCES executables/TestPseudoModelState4D.cc + ARGS "testinput/pseudomodel_state4d.yaml" + LIBS lorenz95 + TEST_DEPENDS test_l95_truth ) + ecbuild_add_test( TARGET test_l95_identitymodel SOURCES executables/TestModel.cc ARGS "testinput/identitymodel.yaml" @@ -472,10 +481,10 @@ ecbuild_add_test( TARGET test_l95_3dvar ARGS testinput/3dvar.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) -#ecbuild_add_test( TESTNAME 3dvar_noobs -# COMMAND l95_4dvar.x -# ARGS testinput/3dvar_noobs.yaml -# TEST_DEPENDS test_l95_forecast ) +ecbuild_add_test( TARGET test_3dvar_noobs + COMMAND l95_4dvar.x + ARGS testinput/3dvar_noobs.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) ecbuild_add_test( TARGET test_l95_3dvar_qc COMMAND l95_4dvar.x @@ -502,91 +511,96 @@ ecbuild_add_test( TARGET test_l95_3dfgat # 4d variational tests ##################################################################### -ecbuild_add_test( TARGET test_l95_4dvar.drgmresr +ecbuild_add_test( TARGET test_l95_4dvar_drgmresr COMMAND l95_4dvar.x ARGS testinput/4dvar.drgmresr.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.dripcg +ecbuild_add_test( TARGET test_l95_4dvar_dripcg COMMAND l95_4dvar.x ARGS testinput/4dvar.dripcg.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.dripcgqn +ecbuild_add_test( TARGET test_l95_4dvar_dripcgqn COMMAND l95_4dvar.x ARGS testinput/4dvar.dripcgqn.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.drpcg +ecbuild_add_test( TARGET test_l95_4dvar_drpcg COMMAND l95_4dvar.x ARGS testinput/4dvar.drpcg.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.drpcgqn +ecbuild_add_test( TARGET test_l95_4dvar_drpcgqn COMMAND l95_4dvar.x ARGS testinput/4dvar.drpcgqn.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.drplanczos +ecbuild_add_test( TARGET test_l95_4dvar_drplanczos COMMAND l95_4dvar.x ARGS testinput/4dvar.drplanczos.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.drplanclmp +ecbuild_add_test( TARGET test_l95_4dvar_drplanclmp COMMAND l95_4dvar.x ARGS testinput/4dvar.drplanclmp.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.fgmres +ecbuild_add_test( TARGET test_l95_4dvar_fgmres COMMAND l95_4dvar.x ARGS testinput/4dvar.fgmres.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.gmresr +ecbuild_add_test( TARGET test_l95_4dvar_gmresr COMMAND l95_4dvar.x ARGS testinput/4dvar.gmresr.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.ipcg +ecbuild_add_test( TARGET test_l95_4dvar_ipcg COMMAND l95_4dvar.x ARGS testinput/4dvar.ipcg.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.lbgmresr +ecbuild_add_test( TARGET test_l95_4dvar_lbgmresr COMMAND l95_4dvar.x ARGS testinput/4dvar.lbgmresr.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.pcg +ecbuild_add_test( TARGET test_l95_4dvar_pcg COMMAND l95_4dvar.x ARGS testinput/4dvar.pcg.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.planczos +ecbuild_add_test( TARGET test_l95_4dvar_planczos COMMAND l95_4dvar.x ARGS testinput/4dvar.planczos.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.rpcg +ecbuild_add_test( TARGET test_l95_4dvar_rpcg COMMAND l95_4dvar.x ARGS testinput/4dvar.rpcg.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.rplanczos +ecbuild_add_test( TARGET test_l95_4dvar_rplanczos COMMAND l95_4dvar.x ARGS testinput/4dvar.rplanczos.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.minres +ecbuild_add_test( TARGET test_l95_4dvar_minres COMMAND l95_4dvar.x ARGS testinput/4dvar.minres.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.modbias +ecbuild_add_test( TARGET test_l95_4dvar_modbias COMMAND l95_4dvar.x ARGS testinput/4dvar.modbias.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) +ecbuild_add_test( TARGET test_l95_4dvar_drplzero + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drplzero.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + #ecbuild_add_test( TARGET test_l95_4dforcing # COMMAND l95_4dvar.x # ARGS testinput/4dforcing.yaml @@ -598,25 +612,22 @@ ecbuild_add_test( TARGET test_l95_4dsaddlepoint ARGS testinput/4dsaddlepoint.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -ecbuild_add_test( TARGET test_l95_4dvar.obsbias +ecbuild_add_test( TARGET test_l95_4dvar_obsbias COMMAND l95_4dvar.x ARGS testinput/4dvar.obsbias.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) -#-------------------------------------------------------------------- - -ecbuild_add_test( TARGET test_l95_4dvar.allbiases +ecbuild_add_test( TARGET test_l95_4dvar_allbiases COMMAND l95_4dvar.x ARGS testinput/4dvar.allbiases.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) -#-------------------------------------------------------------------- -ecbuild_add_test( TARGET test_l95_4dvar.alpha +ecbuild_add_test( TARGET test_l95_4dvar_alpha COMMAND l95_4dvar.x ARGS testinput/4dvar.alpha.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) -ecbuild_add_test( TARGET test_l95_4dvar.hybrid +ecbuild_add_test( TARGET test_l95_4dvar_hybrid COMMAND l95_4dvar.x ARGS testinput/4dvar.hybrid.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) @@ -627,7 +638,7 @@ ecbuild_add_test( TARGET test_l95_4densvar ARGS testinput/4densvar.yaml TEST_DEPENDS test_l95_forecast test_l95_makeobs4d12h test_l95_genenspert ) -ecbuild_add_test( TARGET test_l95_4densvar.hybrid +ecbuild_add_test( TARGET test_l95_4densvar_hybrid COMMAND l95_4dvar.x MPI 9 ARGS testinput/4densvar.hybrid.yaml @@ -770,7 +781,6 @@ ecbuild_add_test( TARGET test_l95_simplifiedl95_DRIPCG COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_DRIPCG.yaml ) - ecbuild_add_test( TARGET test_l95_simplifiedl95_DRPCG COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_DRPCG.yaml ) @@ -779,42 +789,42 @@ ecbuild_add_test( TARGET test_l95_simplifiedl95_DRPFOM COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_DRPFOM.yaml ) -ecbuild_add_test( TARGET simplifiedl95_DRPLanczos +ecbuild_add_test( TARGET test_l95_simplifiedl95_DRPLanczos COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_DRPLanczos.yaml ) -ecbuild_add_test( TARGET simplifiedl95_FGMRES +ecbuild_add_test( TARGET test_l95_simplifiedl95_FGMRES COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_FGMRES.yaml ) -ecbuild_add_test( TARGET simplifiedl95_GMRESR +ecbuild_add_test( TARGET test_l95_simplifiedl95_GMRESR COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_GMRESR.yaml ) -ecbuild_add_test( TARGET simplifiedl95_IPCG +ecbuild_add_test( TARGET test_l95_simplifiedl95_IPCG COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_IPCG.yaml ) -ecbuild_add_test( TARGET simplifiedl95_LBGMRESR +ecbuild_add_test( TARGET test_l95_simplifiedl95_LBGMRESR COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_LBGMRESR.yaml ) -ecbuild_add_test( TARGET simplifiedl95_MINRES +ecbuild_add_test( TARGET test_l95_simplifiedl95_MINRES COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_MINRES.yaml ) -ecbuild_add_test( TARGET simplifiedl95_PCG +ecbuild_add_test( TARGET test_l95_simplifiedl95_PCG COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_PCG.yaml ) -ecbuild_add_test( TARGET simplifiedl95_PLanczos +ecbuild_add_test( TARGET test_l95_simplifiedl95_PLanczos COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_PLanczos.yaml ) -ecbuild_add_test( TARGET simplifiedl95_RPCG +ecbuild_add_test( TARGET test_l95_simplifiedl95_RPCG COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_RPCG.yaml ) -ecbuild_add_test( TARGET simplifiedl95_RPLanczos +ecbuild_add_test( TARGET test_l95_simplifiedl95_RPLanczos COMMAND l95_4dvar.x ARGS testinput/simplifiedl95_RPLanczos.yaml ) diff --git a/l95/test/executables/TestPseudoModelState4D.cc b/l95/test/executables/TestPseudoModelState4D.cc new file mode 100644 index 000000000..b566f9959 --- /dev/null +++ b/l95/test/executables/TestPseudoModelState4D.cc @@ -0,0 +1,19 @@ +/* + * (C) Copyright 2021- UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/L95Traits.h" +#include "oops/generic/instantiateModelFactory.h" +#include "oops/runs/Run.h" +#include "test/generic/PseudoModelState4D.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + oops::instantiateModelFactory(); + test::PseudoModelState4D tests; + return run.execute(tests); +} + diff --git a/l95/test/lorenz95/LinearModelFactory.cc b/l95/test/lorenz95/LinearModelFactory.cc index 493a0900f..7b066959e 100644 --- a/l95/test/lorenz95/LinearModelFactory.cc +++ b/l95/test/lorenz95/LinearModelFactory.cc @@ -11,7 +11,7 @@ #include "eckit/testing/Test.h" #include "lorenz95/L95Traits.h" #include "lorenz95/LocsL95.h" -#include "oops/interface/LinearModel.h" +#include "oops/generic/LinearModelBase.h" #include "oops/runs/Run.h" #include "oops/runs/Test.h" #include "oops/util/Expect.h" @@ -31,7 +31,7 @@ CASE("test_linearmodelparameterswrapper_invalid_name") { if (oops::Parameters::isValidationSupported()) EXPECT_THROWS_MSG(parameters.validate(config), "unrecognized enum value"); EXPECT_THROWS_MSG(parameters.deserialize(config), - "does not exist in the tangent linear model factory"); + "does not exist in the linear model factory"); } CASE("test_linearmodelfactory") { diff --git a/l95/test/testinput/3dfgat.yaml b/l95/test/testinput/3dfgat.yaml index 5e3c8ec0d..2fa585cbe 100644 --- a/l95/test/testinput/3dfgat.yaml +++ b/l95/test/testinput/3dfgat.yaml @@ -33,7 +33,8 @@ variational: ninner: 10 gradient norm reduction: 1e-10 linear model: - name: identity + name: Identity + increment variables: [x] variable change: Identity tstep: PT6H diagnostics: diff --git a/l95/test/testinput/3dvar.yaml b/l95/test/testinput/3dvar.yaml index 80bdd36dc..9e195944c 100644 --- a/l95/test/testinput/3dvar.yaml +++ b/l95/test/testinput/3dvar.yaml @@ -33,11 +33,11 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-02T00:00:00Z - exp: test.iter1 - type: in + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: test.iter1 + type: in - ninner: 10 gradient norm reduction: 1e-10 geometry: @@ -45,11 +45,11 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-02T00:00:00Z - exp: test.iter2 - type: in + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: test.iter2 + type: in final: diagnostics: departures: oman diff --git a/l95/test/testinput/3dvar_4fsoi_dripcg.yaml b/l95/test/testinput/3dvar_4fsoi_dripcg.yaml index 77eb9b43b..e7936bd5c 100644 --- a/l95/test/testinput/3dvar_4fsoi_dripcg.yaml +++ b/l95/test/testinput/3dvar_4fsoi_dripcg.yaml @@ -33,11 +33,11 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-02T00:00:00Z - exp: 3dvar_4fsoi_dripcg.iter1 - type: in + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: 3dvar_4fsoi_dripcg.iter1 + type: in final: diagnostics: departures: oman diff --git a/l95/test/testinput/3dvar_4fsoi_pcg.yaml b/l95/test/testinput/3dvar_4fsoi_pcg.yaml index 3fd95c637..9d3cfd66d 100644 --- a/l95/test/testinput/3dvar_4fsoi_pcg.yaml +++ b/l95/test/testinput/3dvar_4fsoi_pcg.yaml @@ -33,11 +33,11 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-02T00:00:00Z - exp: 3dvar_4fsoi_pcg.iter1 - type: in + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: 3dvar_4fsoi_pcg.iter1 + type: in final: diagnostics: departures: oman diff --git a/l95/test/testinput/3dvar_noobs.yaml b/l95/test/testinput/3dvar_noobs.yaml index 4a21a8046..f28f81ae6 100644 --- a/l95/test/testinput/3dvar_noobs.yaml +++ b/l95/test/testinput/3dvar_noobs.yaml @@ -21,7 +21,7 @@ cost function: covariance model: diagonal variational: minimizer: - algorithm: DRIPCG + algorithm: RPCG iterations: - ninner: 10 gradient norm reduction: 1e-10 diff --git a/l95/test/testinput/3dvar_qc_obserr.yaml b/l95/test/testinput/3dvar_qc_obserr.yaml index 15186552d..951d502f5 100644 --- a/l95/test/testinput/3dvar_qc_obserr.yaml +++ b/l95/test/testinput/3dvar_qc_obserr.yaml @@ -8,7 +8,6 @@ cost function: name: L95 f: 8.0 tstep: PT1H30M - variable change: Identity analysis variables: [x] background: date: 2010-01-02T00:00:00Z diff --git a/l95/test/testinput/4dvar.drplanczos.yaml b/l95/test/testinput/4dvar.drplanczos.yaml index f9fe1698a..078f83863 100644 --- a/l95/test/testinput/4dvar.drplanczos.yaml +++ b/l95/test/testinput/4dvar.drplanczos.yaml @@ -34,11 +34,11 @@ variational: algorithm: DRPLanczos online diagnostics: write basis: true - krylov basis: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: 4dvar.drplanczos - type: krylov + krylov basis: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 4dvar.drplanczos + type: krylov iterations: - ninner: 10 gradient norm reduction: 1.0e-10 diff --git a/l95/test/testinput/4dvar.drplzero.yaml b/l95/test/testinput/4dvar.drplzero.yaml new file mode 100644 index 000000000..834e161a9 --- /dev/null +++ b/l95/test/testinput/4dvar.drplzero.yaml @@ -0,0 +1,78 @@ +cost function: + cost type: 4D-Var + window begin: 2010-01-01T03:00:00Z + window length: P1D + geometry: + resol: 40 + model: + name: L95 + f: 8.0 + tstep: PT1H30M + analysis variables: [x] + background: + date: 2010-01-01T03:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.PT3H + background error: + covariance model: L95Error + date: 2010-01-01T03:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/l95.truth4d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.4dvar.drplanczos.2010-01-02T00:00:00Z.obt + monitoring only: true + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: DRPLanczos + online diagnostics: + write basis: true + krylov basis: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 4dvar.drplanczos + type: krylov + iterations: + - ninner: 10 + gradient norm reduction: 1.0e-10 + geometry: + resol: 40 + linear model: + name: L95TLM + tstep: PT1H30M + trajectory: + f: 8.0 + tstep: PT1H30M + variable change: Identity + diagnostics: + departures: ombg + test: on + - ninner: 10 + gradient norm reduction: 1.0e-10 + geometry: + resol: 40 + linear model: + name: L95TLM + tstep: PT1H30M + trajectory: + f: 8.0 + tstep: PT1H30M + variable change: Identity + test: on +final: + diagnostics: + departures: oman + prints: + frequency: PT1H30M +output: + datadir: Data + exp: test + first: PT3H + frequency: PT06H + type: an + +test: + reference filename: testoutput/4dvar.drplzero.test diff --git a/l95/test/testinput/eda.3dfgat.1.yaml b/l95/test/testinput/eda.3dfgat.1.yaml index 3ed0a63ea..8961b13eb 100644 --- a/l95/test/testinput/eda.3dfgat.1.yaml +++ b/l95/test/testinput/eda.3dfgat.1.yaml @@ -35,7 +35,8 @@ variational: ninner: 10 gradient norm reduction: 1e-10 linear model: - name: identity + name: Identity + increment variables: [x] variable change: Identity tstep: PT6H diagnostics: diff --git a/l95/test/testinput/eda.3dfgat.2.yaml b/l95/test/testinput/eda.3dfgat.2.yaml index cb80db463..0ad8c2faf 100644 --- a/l95/test/testinput/eda.3dfgat.2.yaml +++ b/l95/test/testinput/eda.3dfgat.2.yaml @@ -35,7 +35,8 @@ variational: ninner: 10 gradient norm reduction: 1e-10 linear model: - name: identity + name: Identity + increment variables: [x] variable change: Identity tstep: PT6H diagnostics: diff --git a/l95/test/testinput/eda.3dfgat.3.yaml b/l95/test/testinput/eda.3dfgat.3.yaml index d8a3db094..7e211ce98 100644 --- a/l95/test/testinput/eda.3dfgat.3.yaml +++ b/l95/test/testinput/eda.3dfgat.3.yaml @@ -35,7 +35,8 @@ variational: ninner: 10 gradient norm reduction: 1e-10 linear model: - name: identity + name: Identity + increment variables: [x] variable change: Identity tstep: PT6H diagnostics: diff --git a/l95/test/testinput/eda.3dfgat.4.yaml b/l95/test/testinput/eda.3dfgat.4.yaml index 1262bbe8d..801cde08b 100644 --- a/l95/test/testinput/eda.3dfgat.4.yaml +++ b/l95/test/testinput/eda.3dfgat.4.yaml @@ -35,7 +35,8 @@ variational: ninner: 10 gradient norm reduction: 1e-10 linear model: - name: identity + name: Identity + increment variables: [x] variable change: Identity tstep: PT6H diagnostics: diff --git a/l95/test/testinput/eda.3dvar.block.1.yaml b/l95/test/testinput/eda.3dvar.block.1.yaml index 8931bcc49..03804536a 100644 --- a/l95/test/testinput/eda.3dvar.block.1.yaml +++ b/l95/test/testinput/eda.3dvar.block.1.yaml @@ -2,7 +2,6 @@ cost function: cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H - variable change: Identity geometry: resol: 40 analysis variables: [x] @@ -29,11 +28,11 @@ variational: members: 2 online diagnostics: write basis: true - krylov basis: - datadir: Data - date: 2010-01-01T00:00:00Z - exp: 3dvar.block.m1 - type: krylov + krylov basis: + datadir: Data + date: 2010-01-01T00:00:00Z + exp: 3dvar.block.m1 + type: krylov iterations: - geometry: resol: 40 diff --git a/l95/test/testinput/eda.3dvar.block.2.yaml b/l95/test/testinput/eda.3dvar.block.2.yaml index c65cffc3e..ca8b93017 100644 --- a/l95/test/testinput/eda.3dvar.block.2.yaml +++ b/l95/test/testinput/eda.3dvar.block.2.yaml @@ -2,7 +2,6 @@ cost function: cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H - variable change: Identity geometry: resol: 40 analysis variables: [x] @@ -29,11 +28,11 @@ variational: members: 2 online diagnostics: write basis: true - krylov basis: - datadir: Data - date: 2010-01-01T00:00:00Z - exp: 3dvar.block.m2 - type: krylov + krylov basis: + datadir: Data + date: 2010-01-01T00:00:00Z + exp: 3dvar.block.m2 + type: krylov iterations: - geometry: resol: 40 diff --git a/l95/test/testinput/getkf.yaml b/l95/test/testinput/getkf.yaml index d8e35dd91..5bfb300b5 100644 --- a/l95/test/testinput/getkf.yaml +++ b/l95/test/testinput/getkf.yaml @@ -49,4 +49,4 @@ output: test: reference filename: testoutput/getkf.test - float relative tolerance: 1e-3 + float relative tolerance: 1.5e-3 diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 36dc15dee..45c29f01a 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -42,6 +42,7 @@ test date: '2010-01-01T03:00:00Z' increment test: date: '2010-01-01T03:00:00Z' + skip atlas: 1 model: f: 8.0 diff --git a/l95/test/testinput/letkf_qc.yaml b/l95/test/testinput/letkf_qc.yaml index b85645ae8..c662bc504 100644 --- a/l95/test/testinput/letkf_qc.yaml +++ b/l95/test/testinput/letkf_qc.yaml @@ -48,3 +48,5 @@ output: test: reference filename: testoutput/letkf_qc.test + log output filename: testoutput/letkf_qc.log.out + test output filename: testoutput/letkf_qc.test.out diff --git a/l95/test/testinput/pseudomodel_state4d.yaml b/l95/test/testinput/pseudomodel_state4d.yaml new file mode 100644 index 000000000..f657604c7 --- /dev/null +++ b/l95/test/testinput/pseudomodel_state4d.yaml @@ -0,0 +1,43 @@ +geometry: + resol: 40 +model: + name: PseudoModel + tstep: PT3H + state variables: ['x'] + states: + - date: '2010-01-01T06:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T09:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T12:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T15:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT15H + - date: '2010-01-01T18:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT18H + - date: '2010-01-01T21:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT21H + - date: '2010-01-02T00:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.P1D +model aux control: {} +initial condition: + date: '2010-01-01T03:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT3H +pseudo model with 4D state: + states: + - date: '2010-01-01T03:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T06:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T09:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T12:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T15:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT15H + - date: '2010-01-01T18:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT18H + - date: '2010-01-01T21:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.PT21H + - date: '2010-01-02T00:00:00Z' + filename: Data/truth.fc.2010-01-01T00:00:00Z.P1D diff --git a/l95/test/testinput/simplifiedl95_DRGMRESR.yaml b/l95/test/testinput/simplifiedl95_DRGMRESR.yaml index 30a628d94..ee5509762 100644 --- a/l95/test/testinput/simplifiedl95_DRGMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_DRGMRESR.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: DRGMRESR diff --git a/l95/test/testinput/simplifiedl95_DRIPCG.yaml b/l95/test/testinput/simplifiedl95_DRIPCG.yaml index 606068c5e..8025f819b 100644 --- a/l95/test/testinput/simplifiedl95_DRIPCG.yaml +++ b/l95/test/testinput/simplifiedl95_DRIPCG.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: DRIPCG diff --git a/l95/test/testinput/simplifiedl95_DRPCG.yaml b/l95/test/testinput/simplifiedl95_DRPCG.yaml index 42809a453..64fa8aa22 100644 --- a/l95/test/testinput/simplifiedl95_DRPCG.yaml +++ b/l95/test/testinput/simplifiedl95_DRPCG.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: DRPCG diff --git a/l95/test/testinput/simplifiedl95_DRPFOM.yaml b/l95/test/testinput/simplifiedl95_DRPFOM.yaml index 286c6f62b..a05c9f768 100644 --- a/l95/test/testinput/simplifiedl95_DRPFOM.yaml +++ b/l95/test/testinput/simplifiedl95_DRPFOM.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: DRPFOM diff --git a/l95/test/testinput/simplifiedl95_DRPLanczos.yaml b/l95/test/testinput/simplifiedl95_DRPLanczos.yaml index a11276af4..4ea106edc 100644 --- a/l95/test/testinput/simplifiedl95_DRPLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_DRPLanczos.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: DRPLanczos diff --git a/l95/test/testinput/simplifiedl95_FGMRES.yaml b/l95/test/testinput/simplifiedl95_FGMRES.yaml index e8023d6f8..e8e9ff49f 100644 --- a/l95/test/testinput/simplifiedl95_FGMRES.yaml +++ b/l95/test/testinput/simplifiedl95_FGMRES.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: FGMRES diff --git a/l95/test/testinput/simplifiedl95_GMRESR.yaml b/l95/test/testinput/simplifiedl95_GMRESR.yaml index 58e172ad7..c2d381b15 100644 --- a/l95/test/testinput/simplifiedl95_GMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_GMRESR.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: GMRESR diff --git a/l95/test/testinput/simplifiedl95_IPCG.yaml b/l95/test/testinput/simplifiedl95_IPCG.yaml index 19c975543..c137cb48f 100644 --- a/l95/test/testinput/simplifiedl95_IPCG.yaml +++ b/l95/test/testinput/simplifiedl95_IPCG.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: IPCG diff --git a/l95/test/testinput/simplifiedl95_LBGMRESR.yaml b/l95/test/testinput/simplifiedl95_LBGMRESR.yaml index 11d7d8840..8d77541dc 100644 --- a/l95/test/testinput/simplifiedl95_LBGMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_LBGMRESR.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: LBGMRESR diff --git a/l95/test/testinput/simplifiedl95_MINRES.yaml b/l95/test/testinput/simplifiedl95_MINRES.yaml index a6e1aeb52..7be77e8a1 100644 --- a/l95/test/testinput/simplifiedl95_MINRES.yaml +++ b/l95/test/testinput/simplifiedl95_MINRES.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: MINRES diff --git a/l95/test/testinput/simplifiedl95_PCG.yaml b/l95/test/testinput/simplifiedl95_PCG.yaml index fe0c15317..63d69b5d8 100644 --- a/l95/test/testinput/simplifiedl95_PCG.yaml +++ b/l95/test/testinput/simplifiedl95_PCG.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: PCG diff --git a/l95/test/testinput/simplifiedl95_PLanczos.yaml b/l95/test/testinput/simplifiedl95_PLanczos.yaml index e27fa6d61..ff4214ba3 100644 --- a/l95/test/testinput/simplifiedl95_PLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_PLanczos.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: PLanczos diff --git a/l95/test/testinput/simplifiedl95_RPCG.yaml b/l95/test/testinput/simplifiedl95_RPCG.yaml index b7ab16f23..eebda84da 100644 --- a/l95/test/testinput/simplifiedl95_RPCG.yaml +++ b/l95/test/testinput/simplifiedl95_RPCG.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: RPCG diff --git a/l95/test/testinput/simplifiedl95_RPLanczos.yaml b/l95/test/testinput/simplifiedl95_RPLanczos.yaml index f82bcf6bd..80e9ce2f1 100644 --- a/l95/test/testinput/simplifiedl95_RPLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_RPLanczos.yaml @@ -24,7 +24,7 @@ cost function: obsdataout: Data/simplifiedl95.obt obs error: covariance model: diagonal - standard_deviation: 1.0 + random amplitude: 1.0 variational: minimizer: algorithm: RPLanczos diff --git a/l95/test/testoutput/3dvar_noobs.test b/l95/test/testoutput/3dvar_noobs.test index f5053155d..f9c8ab222 100644 --- a/l95/test/testoutput/3dvar_noobs.test +++ b/l95/test/testoutput/3dvar_noobs.test @@ -1,17 +1,21 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations -Test : CostFunction: Nonlinear J = 0 -Test : DRIPCGMinimizer: reduction in residual norm = 1 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.65953, Max=9.39191, Average=7.97085 -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations -Test : CostFunction: Nonlinear J = 0 -Test : DRIPCGMinimizer: reduction in residual norm = 1 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.65953, Max=9.39191, Average=7.97085 -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations -Test : CostFunction: Nonlinear J = 0 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +CostFunction: Nonlinear J = 0 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.65953, Max=9.39191, Average=7.97085 + + + +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +CostFunction: Nonlinear J = 0 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.65953, Max=9.39191, Average=7.97085 + + + +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +CostFunction: Nonlinear J = 0 diff --git a/l95/test/testoutput/4dvar.drplzero.test b/l95/test/testoutput/4dvar.drplzero.test new file mode 100644 index 000000000..b18538a07 --- /dev/null +++ b/l95/test/testoutput/4dvar.drplzero.test @@ -0,0 +1,21 @@ +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 (Monitoring only) +CostFunction: Nonlinear J = 0 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.0282, Max=8.19498, Average=7.97551 + + + +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 (Monitoring only) +CostFunction: Nonlinear J = 0 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.0282, Max=8.19498, Average=7.97551 + + + +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 (Monitoring only) +CostFunction: Nonlinear J = 0 diff --git a/l95/test/testoutput/letkf.test b/l95/test/testoutput/letkf.test index d7095acef..f36ee8828 100644 --- a/l95/test/testoutput/letkf.test +++ b/l95/test/testoutput/letkf.test @@ -1,52 +1,38 @@ Initial state for member 1: Valid time: 2010-01-02T00:00:00Z Min=3.53609, Max=11.1974, Average=7.90978 - Initial state for member 2: Valid time: 2010-01-02T00:00:00Z Min=3.77507, Max=11.1844, Average=7.75047 - Initial state for member 3: Valid time: 2010-01-02T00:00:00Z Min=5.36882, Max=10.844, Average=7.80576 - Initial state for member 4: Valid time: 2010-01-02T00:00:00Z Min=4.14639, Max=11.6734, Average=7.76001 - Initial state for member 5: Valid time: 2010-01-02T00:00:00Z Min=3.17563, Max=10.8183, Average=7.60243 - H(x) for member 1: Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 - H(x) for member 2: Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 - H(x) for member 3: Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 - H(x) for member 4: Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 - H(x) for member 5: Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 - H(x) ensemble background mean: Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 - background y - H(x): Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 - Background mean : Valid time: 2010-01-02T00:00:00Z Min=5.25199, Max=9.50208, Average=7.76569 - Analysis mean : Valid time: 2010-01-02T00:00:00Z Min=6.73044, Max=9.19002, Average=7.98832 - Analysis mean increment : Valid time: 2010-01-02T00:00:00Z Min=-1.55316, Max=2.41362, Average=0.222628 @@ -58,24 +44,17 @@ Analysis variance : Min=0.216453, Max=8.21437, Average=2.10118 H(x) for member 1: Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 - H(x) for member 2: Lorenz 95 nobs= 120 Min=5.53728, Max=9.94295, Average=7.97522 - H(x) for member 3: Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01708 - H(x) for member 4: Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98557 - H(x) for member 5: Lorenz 95 nobs= 120 Min=5.01152, Max=9.81763, Average=7.86627 - H(x) ensemble analysis mean: Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98832 - analysis y - H(x): Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0230717 - ombg RMS: 0.864096 oman RMS: 0.146322 diff --git a/l95/test/testoutput/letkf_qc.test b/l95/test/testoutput/letkf_qc.test index 561a11a04..2885ea54a 100644 --- a/l95/test/testoutput/letkf_qc.test +++ b/l95/test/testoutput/letkf_qc.test @@ -45,28 +45,28 @@ Background mean : Analysis mean : Valid time: 2010-01-02T00:00:00Z - Min=6.73044, Max=9.19002, Average=7.98752 + Min=6.73044, Max=9.19002, Average=7.98832 H(x) for member 1: -Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09272 +Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 H(x) for member 2: -Lorenz 95 nobs= 120 Min=5.62563, Max=9.94295, Average=7.97729 +Lorenz 95 nobs= 120 Min=5.53728, Max=9.94295, Average=7.97522 H(x) for member 3: -Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01934 +Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01708 H(x) for member 4: -Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98859 +Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98557 H(x) for member 5: -Lorenz 95 nobs= 120 Min=4.72238, Max=9.79569, Average=7.85967 +Lorenz 95 nobs= 120 Min=5.01152, Max=9.81763, Average=7.86627 H(x) ensemble analysis mean: -Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98752 +Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98832 analysis y - H(x): -Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0238667 +Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0230717 ombg RMS: 0.864096 -oman RMS: 0.14948 +oman RMS: 0.146322 diff --git a/qg/ewok/README.md b/qg/ewok/README similarity index 100% rename from qg/ewok/README.md rename to qg/ewok/README diff --git a/qg/mains/qgEnsHofX.cc b/qg/mains/qgEnsHofX.cc index 0de148494..d55bbfd7e 100644 --- a/qg/mains/qgEnsHofX.cc +++ b/qg/mains/qgEnsHofX.cc @@ -9,12 +9,14 @@ */ #include "model/QgTraits.h" +#include "oops/qg/instantiateQgChangeVarFactory.h" #include "oops/runs/EnsembleApplication.h" #include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + qg::instantiateQgChangeVarFactory(); oops::EnsembleApplication< oops::HofX4D > enshofx; return run.execute(enshofx); } diff --git a/qg/mains/qgHofX.cc b/qg/mains/qgHofX.cc index 44775f63c..8f6047022 100644 --- a/qg/mains/qgHofX.cc +++ b/qg/mains/qgHofX.cc @@ -9,11 +9,13 @@ */ #include "model/QgTraits.h" +#include "oops/qg/instantiateQgChangeVarFactory.h" #include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + qg::instantiateQgChangeVarFactory(); oops::HofX4D hofx; return run.execute(hofx); } diff --git a/qg/mains/qgLETKF.cc b/qg/mains/qgLETKF.cc index af1f456b2..21a747fb5 100644 --- a/qg/mains/qgLETKF.cc +++ b/qg/mains/qgLETKF.cc @@ -6,11 +6,13 @@ */ #include "model/QgTraits.h" +#include "oops/qg/instantiateQgChangeVarFactory.h" #include "oops/runs/LocalEnsembleDA.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + qg::instantiateQgChangeVarFactory(); oops::LocalEnsembleDA letkf; return run.execute(letkf); } diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index 8d8cf3878..890818508 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -9,6 +9,8 @@ ErrorCovarianceQG.cc ErrorCovarianceQG.h FieldsQG.cc FieldsQG.h +FinalCheck.cc +FinalCheck.h GeometryQG.cc GeometryQG.h GeometryQGIterator.cc @@ -47,6 +49,7 @@ ObsOpBaseQG.cc ObsOpBaseQG.h ObsOpBaseTLAD.cc ObsOpBaseTLAD.h +ObsOperatorParameters.h ObsOperatorQG.cc ObsOperatorQG.h ObsOperatorTLAD.cc @@ -73,8 +76,6 @@ QgFortran.h QgTraits.h StateQG.cc StateQG.h -TlmIdQG.cc -TlmIdQG.h TlmQG.cc TlmQG.h fft_f.cc diff --git a/qg/model/FieldsQG.cc b/qg/model/FieldsQG.cc index 37652dc4f..466c91e87 100644 --- a/qg/model/FieldsQG.cc +++ b/qg/model/FieldsQG.cc @@ -237,26 +237,25 @@ void FieldsQG::setLocal(const oops::LocalIncrement & x, const GeometryQGIterator } // ----------------------------------------------------------------------------- size_t FieldsQG::serialSize() const { - size_t nn = 0; int nx, ny, nz, lbc; qg_fields_sizes_f90(keyFlds_, nx, ny, nz); qg_fields_lbc_f90(keyFlds_, lbc); - nn += nx * ny * nz; + size_t nn = nx * ny * nz; if (lbc == 1) { - nn += + 2 * (nx + 1) * nz; + nn += 2 * (nx + 1) * nz; } nn += time_.serialSize(); return nn; } // ----------------------------------------------------------------------------- void FieldsQG::serialize(std::vector & vect) const { - int size_fld = this->serialSize() - 2; + size_t size_fld = this->serialSize() - time_.serialSize(); // Allocate space for fld, xb and qb std::vector v_fld(size_fld, 0); // Serialize the field - qg_fields_serialize_f90(keyFlds_, size_fld, v_fld.data()); + qg_fields_serialize_f90(keyFlds_, static_cast(size_fld), v_fld.data()); vect.insert(vect.end(), v_fld.begin(), v_fld.end()); // Serialize the date and time @@ -264,7 +263,9 @@ void FieldsQG::serialize(std::vector & vect) const { } // ----------------------------------------------------------------------------- void FieldsQG::deserialize(const std::vector & vect, size_t & index) { - qg_fields_deserialize_f90(keyFlds_, vect.size(), vect.data(), index); + int indexInt = static_cast(index); + qg_fields_deserialize_f90(keyFlds_, static_cast(vect.size()), vect.data(), indexInt); + index = static_cast(indexInt); time_.deserialize(vect, index); } // ----------------------------------------------------------------------------- diff --git a/qg/model/FinalCheck.cc b/qg/model/FinalCheck.cc new file mode 100644 index 000000000..bd8e081d4 --- /dev/null +++ b/qg/model/FinalCheck.cc @@ -0,0 +1,14 @@ +/* + * (C) Crown Copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "model/FinalCheck.h" + +namespace qg { +// ----------------------------------------------------------------------------- +static oops::interface::FilterMaker makerPreChk_("Final Check"); +// ----------------------------------------------------------------------------- +} // namespace qg diff --git a/qg/model/FinalCheck.h b/qg/model/FinalCheck.h new file mode 100644 index 000000000..0e3b264fe --- /dev/null +++ b/qg/model/FinalCheck.h @@ -0,0 +1,48 @@ +/* + * (C) Crown Copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef QG_MODEL_FINALCHECK_H_ +#define QG_MODEL_FINALCHECK_H_ + +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "model/QgTraits.h" + +#include "oops/base/Variables.h" +#include "oops/interface/ObsFilterBase.h" +#include "oops/util/Printable.h" + +namespace qg { + class GomQG; + template class ObsDataQG; + class ObsDiagsQG; + class ObsSpaceQG; + class ObsVecQG; + +class FinalCheck : public oops::interface::ObsFilterBase { + public: + FinalCheck(const ObsSpaceQG &, const eckit::Configuration &, + std::shared_ptr >, std::shared_ptr >): novars_() {} + + void preProcess() override {} + void priorFilter(const GomQG &) override {} + void postFilter(const ObsVecQG &, const ObsVecQG &, const ObsDiagsQG &) override {} + + oops::Variables requiredVars() const override {return novars_;} + oops::Variables requiredHdiagnostics() const override {return novars_;} + + private: + void print(std::ostream &) const override {} + const oops::Variables novars_; +}; + +} // namespace qg + +#endif // QG_MODEL_FINALCHECK_H_ diff --git a/qg/model/GeometryQG.cc b/qg/model/GeometryQG.cc index 27b8ca544..6db95c257 100644 --- a/qg/model/GeometryQG.cc +++ b/qg/model/GeometryQG.cc @@ -15,6 +15,7 @@ #include "atlas/grid.h" #include "atlas/util/Config.h" +#include "oops/base/Variables.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" @@ -101,7 +102,13 @@ std::vector GeometryQG::verticalCoord(std::string & vcUnits) const { oops::Log::debug() << "QG vert coord: " << vc << std::endl; return vc; } - +// ------------------------------------------------------------------------------------------------- +std::vector GeometryQG::variableSizes(const oops::Variables & vars) const { + // Note: in qg we always do trilinear interpolation, so GeoVaLs are always + // size 1. + std::vector sizes(vars.size(), 1); + return sizes; +} // ----------------------------------------------------------------------------- void GeometryQG::print(std::ostream & os) const { int nx; diff --git a/qg/model/GeometryQG.h b/qg/model/GeometryQG.h index 148339801..08e57c3bc 100644 --- a/qg/model/GeometryQG.h +++ b/qg/model/GeometryQG.h @@ -30,6 +30,10 @@ #include "oops/qg/GeometryQGIterator.h" #include "oops/qg/QgFortran.h" +namespace oops { + class Variables; +} + namespace qg { class GeometryQgParameters : public oops::Parameters { @@ -70,6 +74,8 @@ class GeometryQG : public util::Printable, atlas::FunctionSpace * atlasFunctionSpace() const {return atlasFunctionSpace_.get();} atlas::FieldSet * atlasFieldSet() const {return atlasFieldSet_.get();} + std::vector variableSizes(const oops::Variables & vars) const; + private: GeometryQG & operator=(const GeometryQG &); void print(std::ostream &) const; diff --git a/qg/model/GomQG.cc b/qg/model/GomQG.cc index b4532f719..b3729b8f7 100644 --- a/qg/model/GomQG.cc +++ b/qg/model/GomQG.cc @@ -22,7 +22,8 @@ namespace qg { // ----------------------------------------------------------------------------- -GomQG::GomQG(const LocationsQG & locs, const oops::Variables & vars): +GomQG::GomQG(const LocationsQG & locs, const oops::Variables & vars, + const std::vector & sizes): vars_(vars) { // gom_setup just creates and allocates the GeoVaLs object without filling diff --git a/qg/model/GomQG.h b/qg/model/GomQG.h index 650e3cce5..d174798a6 100644 --- a/qg/model/GomQG.h +++ b/qg/model/GomQG.h @@ -36,7 +36,7 @@ class GomQG : public util::Printable, public: static const std::string classname() {return "qg::GomQG";} - GomQG(const LocationsQG &, const oops::Variables &); + GomQG(const LocationsQG &, const oops::Variables &, const std::vector &); GomQG(const eckit::Configuration &, const ObsSpaceQG &, const oops::Variables &); explicit GomQG(const GomQG &); diff --git a/qg/model/LocalizationMatrixQG.h b/qg/model/LocalizationMatrixQG.h index a6680f63a..5a3c08581 100644 --- a/qg/model/LocalizationMatrixQG.h +++ b/qg/model/LocalizationMatrixQG.h @@ -18,8 +18,7 @@ #include "eckit/config/Configuration.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" +#include "oops/interface/LocalizationBase.h" #include "oops/qg/GeometryQG.h" #include "oops/qg/QgFortran.h" @@ -33,16 +32,15 @@ namespace qg { /// Localization matrix for QG model. // ----------------------------------------------------------------------------- -class LocalizationMatrixQG: public util::Printable, - private util::ObjectCounter { +class LocalizationMatrixQG: public oops::interface::LocalizationBase { public: static const std::string classname() {return "qg::LocalizationMatrixQG";} LocalizationMatrixQG(const GeometryQG &, const eckit::Configuration &); ~LocalizationMatrixQG(); - void randomize(IncrementQG &) const; - void multiply(IncrementQG &) const; + void randomize(IncrementQG &) const override; + void multiply(IncrementQG &) const override; private: void print(std::ostream &) const override; diff --git a/qg/model/ModelQG.cc b/qg/model/ModelQG.cc index 8eb3caecc..2ac4aa6e1 100644 --- a/qg/model/ModelQG.cc +++ b/qg/model/ModelQG.cc @@ -24,7 +24,7 @@ namespace qg { // ----------------------------------------------------------------------------- -static oops::ModelMaker makermodel_("QG"); +static oops::interface::ModelMaker makermodel_("QG"); // ----------------------------------------------------------------------------- ModelQG::ModelQG(const GeometryQG & resol, const ModelQgParameters & params) : keyConfig_(0), params_(params), geom_(resol), diff --git a/qg/model/ModelQG.h b/qg/model/ModelQG.h index 92cba8fdd..5348b4ab8 100644 --- a/qg/model/ModelQG.h +++ b/qg/model/ModelQG.h @@ -16,9 +16,9 @@ #include #include -#include "oops/base/ModelBase.h" #include "oops/base/ParameterTraitsVariables.h" #include "oops/base/Variables.h" +#include "oops/interface/ModelBase.h" #include "oops/util/Duration.h" #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/Parameter.h" @@ -56,7 +56,7 @@ class ModelQgParameters : public oops::ModelParametersBase { * QG nonlinear model definition and configuration parameters. */ -class ModelQG: public oops::ModelBase, +class ModelQG: public oops::interface::ModelBase, private util::ObjectCounter { public: typedef ModelQgParameters Parameters_; diff --git a/qg/model/ObsBias.cc b/qg/model/ObsBias.cc index c5de85e15..d8ed26694 100644 --- a/qg/model/ObsBias.cc +++ b/qg/model/ObsBias.cc @@ -96,7 +96,7 @@ void ObsBias::print(std::ostream & os) const { strs << bias_[jj]; strn += strs.str(); } - os << std::endl << "ObsBias = " << strn; + os << "ObsBias = " << strn; } } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index ed04179af..16fcca025 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -42,8 +42,6 @@ class ObsDataQG : public util::Printable, /// set all values to zero void zero(); - /// set \p i-th value to zero - void zero(int i); /// set all values to one void ones(); void mask(const ObsDataQG); @@ -86,11 +84,6 @@ void ObsDataQG::zero() { } // ----------------------------------------------------------------------------- template -void ObsDataQG::zero(int i) { - data_.zero(i); -} -// ----------------------------------------------------------------------------- -template void ObsDataQG::ones() { data_.ones(); } diff --git a/qg/model/ObsLocQG.cc b/qg/model/ObsLocQG.cc index b566dfacd..e47028cd4 100644 --- a/qg/model/ObsLocQG.cc +++ b/qg/model/ObsLocQG.cc @@ -37,18 +37,18 @@ ObsLocQG::ObsLocQG(const eckit::Configuration & conf, const ObsSpaceQG & obsdb) // ----------------------------------------------------------------------------- void ObsLocQG::computeLocalization(const GeometryQGIterator & p, - ObsDataQG & outside, ObsVecQG &) const { + ObsVecQG & local) const { std::unique_ptr locs = obsdb_.locations(); atlas::Field field_lonlat = locs->lonlat(); auto lonlat = make_view(field_lonlat); eckit::geometry::Point2 refPoint = *p; - outside.ones(); + local.ones(); for (int jj = 0; jj < locs->size(); ++jj) { eckit::geometry::Point2 obsPoint(lonlat(jj, 0), lonlat(jj, 1)); double localDist = eckit::geometry::Sphere::distance(6.371e6, refPoint, obsPoint); - if (localDist < lengthscale_) { - outside.zero(jj); + if (localDist > lengthscale_) { + local.setToMissing(jj); } } } diff --git a/qg/model/ObsLocQG.h b/qg/model/ObsLocQG.h index aad101be2..bb3294f6f 100644 --- a/qg/model/ObsLocQG.h +++ b/qg/model/ObsLocQG.h @@ -12,7 +12,6 @@ #include "oops/base/ObsLocalizationBase.h" -#include "oops/qg/ObsDataQG.h" #include "oops/qg/QgTraits.h" namespace eckit { @@ -30,7 +29,7 @@ class ObsLocQG : public oops::ObsLocalizationBase { public: ObsLocQG(const eckit::Configuration &, const ObsSpaceQG &); - void computeLocalization(const GeometryQGIterator &, ObsDataQG &, ObsVecQG &) const override; + void computeLocalization(const GeometryQGIterator &, ObsVecQG &) const override; private: void print(std::ostream &) const override; diff --git a/qg/model/ObsOperatorParameters.h b/qg/model/ObsOperatorParameters.h new file mode 100644 index 000000000..351edd752 --- /dev/null +++ b/qg/model/ObsOperatorParameters.h @@ -0,0 +1,27 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef QG_MODEL_OBSOPERATORPARAMETERS_H_ +#define QG_MODEL_OBSOPERATORPARAMETERS_H_ + +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/Parameters.h" + +namespace qg { + +/// Parameters controlling the observation operator for the QG model. +class ObservationParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObservationParameters, Parameters); + public: + oops::ConfigurationParameter config{this}; +}; + +// ----------------------------------------------------------------------------- + +} // namespace qg + +#endif // QG_MODEL_OBSOPERATORPARAMETERS_H_ diff --git a/qg/model/ObsOperatorQG.cc b/qg/model/ObsOperatorQG.cc index 3d14a46ee..a7ce27490 100644 --- a/qg/model/ObsOperatorQG.cc +++ b/qg/model/ObsOperatorQG.cc @@ -24,8 +24,8 @@ namespace qg { // ----------------------------------------------------------------------------- -ObsOperatorQG::ObsOperatorQG(const ObsSpaceQG & os, const eckit::Configuration & conf) - : oper_(ObsOpFactory::create(os, conf)) +ObsOperatorQG::ObsOperatorQG(const ObsSpaceQG & os, const Parameters_ & params) + : oper_(ObsOpFactory::create(os, params.config)) {} // ----------------------------------------------------------------------------- @@ -35,7 +35,7 @@ ObsOperatorQG::~ObsOperatorQG() {} // ----------------------------------------------------------------------------- void ObsOperatorQG::simulateObs(const GomQG & gvals, ObsVecQG & yy, const ObsBias & bias, - ObsDiagsQG &) const { + ObsVecQG &, ObsDiagsQG &) const { oper_->simulateObs(gvals, yy, bias); } diff --git a/qg/model/ObsOperatorQG.h b/qg/model/ObsOperatorQG.h index d03cb0275..aa7df95fa 100644 --- a/qg/model/ObsOperatorQG.h +++ b/qg/model/ObsOperatorQG.h @@ -16,14 +16,12 @@ #include +#include "model/ObsOperatorParameters.h" + #include "oops/base/Variables.h" #include "oops/util/Printable.h" // Forward declarations -namespace eckit { - class Configuration; -} - namespace qg { class GomQG; class LocationsQG; @@ -38,11 +36,13 @@ namespace qg { class ObsOperatorQG : public util::Printable, private boost::noncopyable { public: - ObsOperatorQG(const ObsSpaceQG &, const eckit::Configuration &); + typedef ObservationParameters Parameters_; + + ObsOperatorQG(const ObsSpaceQG &, const Parameters_ &); ~ObsOperatorQG(); /// Obs Operator - void simulateObs(const GomQG &, ObsVecQG &, const ObsBias &, ObsDiagsQG &) const; + void simulateObs(const GomQG &, ObsVecQG &, const ObsBias &, ObsVecQG &, ObsDiagsQG &) const; /// Other const oops::Variables & requiredVars() const; // Required input requiredVars from Model diff --git a/qg/model/ObsOperatorTLAD.cc b/qg/model/ObsOperatorTLAD.cc index 9eedf5106..fb925f2f2 100644 --- a/qg/model/ObsOperatorTLAD.cc +++ b/qg/model/ObsOperatorTLAD.cc @@ -23,8 +23,8 @@ namespace qg { // ----------------------------------------------------------------------------- -ObsOperatorTLAD::ObsOperatorTLAD(const ObsSpaceQG & os, const eckit::Configuration & conf) - : oper_(ObsOpTLADFactory::create(os, conf)) +ObsOperatorTLAD::ObsOperatorTLAD(const ObsSpaceQG & os, const Parameters_ & params) + : oper_(ObsOpTLADFactory::create(os, params.config)) {} // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsOperatorTLAD.h b/qg/model/ObsOperatorTLAD.h index f5820def5..3a6ce29b5 100644 --- a/qg/model/ObsOperatorTLAD.h +++ b/qg/model/ObsOperatorTLAD.h @@ -16,12 +16,11 @@ #include +#include "model/ObsOperatorParameters.h" + #include "oops/util/Printable.h" // Forward declarations -namespace eckit { - class Configuration; -} namespace oops { class Variables; @@ -40,7 +39,9 @@ namespace qg { class ObsOperatorTLAD : public util::Printable, private boost::noncopyable { public: - ObsOperatorTLAD(const ObsSpaceQG &, const eckit::Configuration &); + typedef ObservationParameters Parameters_; + + ObsOperatorTLAD(const ObsSpaceQG &, const Parameters_ &); ~ObsOperatorTLAD(); /// Obs Operator diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index 1ff0ce0a5..584271ec3 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -36,22 +36,22 @@ int ObsSpaceQG::theObsFileCount_ = 0; // ----------------------------------------------------------------------------- -ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Comm & comm, +ObsSpaceQG::ObsSpaceQG(const Parameters_ & params, const eckit::mpi::Comm & comm, const util::DateTime & bgn, const util::DateTime & end, const eckit::mpi::Comm & timeComm) - : oops::ObsSpaceBase(config, comm, bgn, end), obsname_(config.getString("obs type")), + : oops::ObsSpaceBase(params, comm, bgn, end), obsname_(params.obsType), winbgn_(bgn), winend_(end), obsvars_() { typedef std::map< std::string, F90odb >::iterator otiter; - eckit::LocalConfiguration fileconf(config); + eckit::LocalConfiguration fileconf = params.toConfiguration(); std::string ofin("-"); - if (config.has("obsdatain")) { - ofin = config.getString("obsdatain.obsfile"); + if (params.obsdatain.value() != boost::none) { + ofin = params.obsdatain.value()->obsfile; } std::string ofout("-"); - if (config.has("obsdataout")) { - ofout = config.getString("obsdataout.obsfile"); + if (params.obsdataout.value() != boost::none) { + ofout = params.obsdataout.value()->obsfile; if (timeComm.size() > 1) { std::ostringstream ss; ss << "_" << timeComm.rank(); @@ -90,11 +90,11 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co } // Generate locations etc... if required - if (config.has("generate")) { - const eckit::LocalConfiguration gconf(config, "generate"); - const util::Duration first(gconf.getString("begin")); + if (params.generate.value() != boost::none) { + const ObsGenerateParameters &gParams = *params.generate.value(); + const util::Duration first(gParams.begin); const util::DateTime start(winbgn_ + first); - const util::Duration freq(gconf.getString("obs_period")); + const util::Duration freq(gParams.obsPeriod); int nobstimes = 0; util::DateTime now(start); while (now <= winend_) { @@ -102,7 +102,7 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co now += freq; } int iobs; - qg_obsdb_generate_f90(key_, obsname_.size(), obsname_.c_str(), gconf, + qg_obsdb_generate_f90(key_, obsname_.size(), obsname_.c_str(), gParams.toConfiguration(), start, freq, nobstimes, iobs); } } diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 0447c530b..2f9397a60 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -24,28 +24,66 @@ #include "oops/base/ObsSpaceBase.h" #include "oops/base/Variables.h" #include "oops/util/DateTime.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" #include "oops/qg/LocationsQG.h" #include "oops/qg/ObsIteratorQG.h" #include "oops/qg/QgFortran.h" -namespace eckit { - class Configuration; -} - namespace qg { class ObsIteratorQG; +/// Contents of the `obsdatain` or `obsdataout` YAML section. +class ObsDataParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsDataParameters, Parameters) + + public: + /// File path. + oops::RequiredParameter obsfile{"obsfile", this}; +}; + +/// Options controlling generation of artificial observations. +class ObsGenerateParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsGenerateParameters, Parameters) + + public: + oops::RequiredParameter begin{"begin", this}; + oops::RequiredParameter obsPeriod{"obs_period", this}; + /// Number of observations to generate in each time slot. + oops::RequiredParameter obsDensity{"obs_density", this}; + oops::RequiredParameter nval{"nval", this}; + oops::RequiredParameter obsError{"obs_error", this}; +}; + +/// \brief Configuration parameters for the QG model's ObsSpace. +class ObsSpaceQGParameters : public oops::ObsSpaceParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsSpaceQGParameters, ObsSpaceParametersBase) + + public: + /// Type of observations. + oops::RequiredParameter obsType{"obs type", this}; + /// File from which to load observations. + oops::OptionalParameter obsdatain{"obsdatain", this}; + /// File to which to save observations and analysis. + oops::OptionalParameter obsdataout{"obsdataout", this}; + /// Options controlling generation of artificial observations. + oops::OptionalParameter generate{"generate", this}; +}; + /// \brief ObsSpace for QG model -// \details ObsSpaceQG is created for each obs type. The underlying Fortran -// structure (key_) is created for each matching input-output filename pair -// (i.e. different obstypes can be stored in the same Fortran structure). -// For mapping between ObsSpaceQG and Fortran structures, -// ObsSpaceQG::theObsFileRegister_ map is used +/// \details ObsSpaceQG is created for each obs type. The underlying Fortran +/// structure (key_) is created for each matching input-output filename pair +/// (i.e. different obstypes can be stored in the same Fortran structure). +/// For mapping between ObsSpaceQG and Fortran structures, +/// ObsSpaceQG::theObsFileRegister_ map is used class ObsSpaceQG : public oops::ObsSpaceBase { public: + typedef ObsSpaceQGParameters Parameters_; + /// create full ObsSpace (read or generate data) - ObsSpaceQG(const eckit::Configuration &, const eckit::mpi::Comm &, + ObsSpaceQG(const Parameters_ &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); ~ObsSpaceQG(); diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index a2d27e42e..3bf4974a5 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -83,8 +83,8 @@ void ObsVecQG::zero() { qg_obsvec_zero_f90(keyOvec_); } // ----------------------------------------------------------------------------- -void ObsVecQG::zero(int ii) { - qg_obsvec_zero_ith_f90(keyOvec_, ii); +void ObsVecQG::setToMissing(int ii) { + qg_obsvec_settomissing_ith_f90(keyOvec_, ii); } // ----------------------------------------------------------------------------- void ObsVecQG::ones() { @@ -126,17 +126,21 @@ void ObsVecQG::mask(const ObsDataQG & mask) { qg_obsvec_mask_f90(keyOvec_, mask.toFortran()); } // ----------------------------------------------------------------------------- +void ObsVecQG::mask(const ObsVecQG & mask) { + qg_obsvec_mask_with_missing_f90(keyOvec_, mask.toFortran()); +} +// ----------------------------------------------------------------------------- void ObsVecQG::save(const std::string & name) const { obsdb_.putdb(name, keyOvec_); } // ----------------------------------------------------------------------------- -Eigen::VectorXd ObsVecQG::packEigen(const ObsDataQG & mask) const { +Eigen::VectorXd ObsVecQG::packEigen(const ObsVecQG & mask) const { Eigen::VectorXd vec(packEigenSize(mask)); qg_obsvec_get_withmask_f90(keyOvec_, mask.toFortran(), vec.data(), vec.size()); return vec; } // ----------------------------------------------------------------------------- -size_t ObsVecQG::packEigenSize(const ObsDataQG & mask) const { +size_t ObsVecQG::packEigenSize(const ObsVecQG & mask) const { int nobs; qg_obsvec_nobs_withmask_f90(keyOvec_, mask.toFortran(), nobs); return nobs; diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index 3445cd67c..008cab229 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -45,14 +45,14 @@ class ObsVecQG : public util::Printable, ObsVecQG & operator*= (const ObsVecQG &); ObsVecQG & operator/= (const ObsVecQG &); - Eigen::VectorXd packEigen(const ObsDataQG &) const; - size_t packEigenSize(const ObsDataQG &) const; + Eigen::VectorXd packEigen(const ObsVecQG &) const; + size_t packEigenSize(const ObsVecQG &) const; size_t size() const; /// set all values to zero void zero(); - /// set \p i-th value to zero - void zero(int i); + /// set \p i-th value to missing value + void setToMissing(int i); /// set all values to one void ones(); void axpy(const double &, const ObsVecQG &); @@ -61,6 +61,7 @@ class ObsVecQG : public util::Printable, double dot_product_with(const ObsVecQG &) const; double rms() const; void mask(const ObsDataQG &); + void mask(const ObsVecQG &); ObsVecQG & operator=(const ObsDataQG &); unsigned int nobs() const; diff --git a/qg/model/QCmanager.cc b/qg/model/QCmanager.cc index 6820948c4..def63ff51 100644 --- a/qg/model/QCmanager.cc +++ b/qg/model/QCmanager.cc @@ -7,16 +7,8 @@ #include "model/QCmanager.h" -#include - -#include "model/ObsDataQG.h" -#include "model/ObsSpaceQG.h" -#include "model/QgTraits.h" -#include "oops/interface/ObsFilter.h" - namespace qg { // ----------------------------------------------------------------------------- -static oops::FilterMaker > makerPreChk_("QCmanager"); +static oops::interface::FilterMaker makerPreChk_("QCmanager"); // ----------------------------------------------------------------------------- } // namespace qg diff --git a/qg/model/QCmanager.h b/qg/model/QCmanager.h index 57e5ea532..80ca6519b 100644 --- a/qg/model/QCmanager.h +++ b/qg/model/QCmanager.h @@ -13,7 +13,10 @@ #include "eckit/config/LocalConfiguration.h" +#include "model/QgTraits.h" + #include "oops/base/Variables.h" +#include "oops/interface/ObsFilterBase.h" #include "oops/util/Printable.h" namespace qg { @@ -23,21 +26,21 @@ namespace qg { class ObsSpaceQG; class ObsVecQG; -class QCmanager : public util::Printable { +class QCmanager : public oops::interface::ObsFilterBase { public: QCmanager(const ObsSpaceQG &, const eckit::Configuration &, std::shared_ptr >, std::shared_ptr >): novars_() {} ~QCmanager() {} - void preProcess() const {} - void priorFilter(const GomQG &) const {} - void postFilter(const ObsVecQG &, const ObsDiagsQG &) const {} + void preProcess() override {} + void priorFilter(const GomQG &) override {} + void postFilter(const ObsVecQG &, const ObsVecQG &, const ObsDiagsQG &) override {} - oops::Variables requiredVars() const {return novars_;} - oops::Variables requiredHdiagnostics() const {return novars_;} + oops::Variables requiredVars() const override {return novars_;} + oops::Variables requiredHdiagnostics() const override {return novars_;} private: - void print(std::ostream &) const {} + void print(std::ostream &) const override {} const oops::Variables novars_; }; diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 883448b5d..c0b140aad 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -119,9 +119,9 @@ extern "C" { atlas::field::FieldSetImpl *); void qg_fields_getpoint_f90(const F90flds&, const F90iter&, const int &, double &); void qg_fields_setpoint_f90(const F90flds&, const F90iter&, const int &, const double &); - void qg_fields_serialize_f90(const F90flds &, const std::size_t &, double[]); - void qg_fields_deserialize_f90(const F90flds &, const std::size_t &, const double[], - const std::size_t &); + void qg_fields_serialize_f90(const F90flds &, const int &, double[]); + void qg_fields_deserialize_f90(const F90flds &, const int &, const double[], int &); + // ----------------------------------------------------------------------------- // GetValues // ----------------------------------------------------------------------------- @@ -216,11 +216,15 @@ extern "C" { void qg_obsvec_delete_f90(F90ovec &); void qg_obsvec_copy_f90(const F90ovec &, const F90ovec &); void qg_obsvec_zero_f90(const F90ovec &); - void qg_obsvec_zero_ith_f90(const F90ovec &, const int &); + void qg_obsvec_settomissing_ith_f90(const F90ovec &, const int &); void qg_obsvec_ones_f90(const F90ovec &); /// set ObsVector (with key \p obsvector_key) values to missing values where /// mask ObsVector (with key \p mask_key) values are set to 1 void qg_obsvec_mask_f90(const F90ovec & obsvector_key, const F90ovec & mask_key); + /// set ObsVector (with key \p obsvector_key) values to missing values where + /// mask ObsVector (with key \p mask_key) values are set to missing value + void qg_obsvec_mask_with_missing_f90(const F90ovec & obsvector_key, + const F90ovec & mask_key); void qg_obsvec_mul_scal_f90(const F90ovec &, const double &); void qg_obsvec_add_f90(const F90ovec &, const F90ovec &); void qg_obsvec_sub_f90(const F90ovec &, const F90ovec &); diff --git a/qg/model/TlmIdQG.cc b/qg/model/TlmIdQG.cc deleted file mode 100644 index 60a0e41eb..000000000 --- a/qg/model/TlmIdQG.cc +++ /dev/null @@ -1,79 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#include "model/TlmIdQG.h" - -#include - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/exception/Exceptions.h" - -#include "oops/util/abor1_cpp.h" -#include "oops/util/Logger.h" - -#include "model/GeometryQG.h" -#include "model/IncrementQG.h" -#include "model/ModelBiasIncrement.h" -#include "model/QgFortran.h" -#include "model/QgTraits.h" -#include "model/StateQG.h" - -namespace qg { -// ----------------------------------------------------------------------------- -static oops::LinearModelMaker makerQGIdTLM_("QgIdTLM"); -// ----------------------------------------------------------------------------- -TlmIdQG::TlmIdQG(const GeometryQG & resol, const eckit::Configuration & tlConf) - : keyConfig_(0), tstep_(), resol_(resol), linvars_({"x"}) -{ - if (tlConf.has("tlm variables")) linvars_ = oops::Variables(tlConf, "tlm variables"); - tstep_ = util::Duration(tlConf.getString("tstep")); - qg_model_setup_f90(keyConfig_, tlConf); - - oops::Log::trace() << "TlmIdQG created" << std::endl; -} -// ----------------------------------------------------------------------------- -TlmIdQG::~TlmIdQG() { - qg_model_delete_f90(keyConfig_); - oops::Log::trace() << "TlmIdQG destructed" << std::endl; -} -// ----------------------------------------------------------------------------- -void TlmIdQG::setTrajectory(const StateQG &, StateQG &, const ModelBias &) {} -// ----------------------------------------------------------------------------- -void TlmIdQG::initializeTL(IncrementQG & dx) const { - ASSERT(dx.fields().isForModel(false)); - oops::Log::debug() << "TlmIdQG::initializeTL" << dx.fields() << std::endl; -} -// ----------------------------------------------------------------------------- -void TlmIdQG::stepTL(IncrementQG & dx, const ModelBiasIncrement &) const { - dx.updateTime(tstep_); -} -// ----------------------------------------------------------------------------- -void TlmIdQG::finalizeTL(IncrementQG & dx) const { - oops::Log::debug() << "TlmIdQG::finalizeTL" << dx.fields() << std::endl; -} -// ----------------------------------------------------------------------------- -void TlmIdQG::initializeAD(IncrementQG & dx) const { - ASSERT(dx.fields().isForModel(false)); - oops::Log::debug() << "TlmIdQG::initializeAD" << dx.fields() << std::endl; -} -// ----------------------------------------------------------------------------- -void TlmIdQG::stepAD(IncrementQG & dx, ModelBiasIncrement &) const { - dx.updateTime(-tstep_); -} -// ----------------------------------------------------------------------------- -void TlmIdQG::finalizeAD(IncrementQG & dx) const { - oops::Log::debug() << "TlmIdQG::finalizeAD" << dx.fields() << std::endl; -} -// ----------------------------------------------------------------------------- -void TlmIdQG::print(std::ostream & os) const { - os << "QG Identity TLM"; -} -// ----------------------------------------------------------------------------- -} // namespace qg diff --git a/qg/model/TlmIdQG.h b/qg/model/TlmIdQG.h deleted file mode 100644 index 07bb6abf5..000000000 --- a/qg/model/TlmIdQG.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef QG_MODEL_TLMIDQG_H_ -#define QG_MODEL_TLMIDQG_H_ - -#include - -#include - -#include "oops/base/LinearModelBase.h" - -#include "oops/base/Variables.h" -#include "oops/util/Duration.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" - -#include "oops/qg/QgTraits.h" - -// Forward declarations -namespace eckit { - class Configuration; -} - -namespace qg { -// ----------------------------------------------------------------------------- -/// QG linear identity model definition. -/*! - * QG linear identity model definition and configuration parameters. - */ - -class TlmIdQG: public oops::LinearModelBase, - private util::ObjectCounter { - public: - static const std::string classname() {return "qg::TlmIdQG";} - - TlmIdQG(const GeometryQG &, const eckit::Configuration &); - ~TlmIdQG(); - -/// Model trajectory computation - void setTrajectory(const StateQG &, StateQG &, const ModelBias &) override; - -/// Run TLM and its adjoint - void initializeTL(IncrementQG &) const override; - void stepTL(IncrementQG &, const ModelBiasIncrement &) const override; - void finalizeTL(IncrementQG &) const override; - - void initializeAD(IncrementQG &) const override; - void stepAD(IncrementQG &, ModelBiasIncrement &) const override; - void finalizeAD(IncrementQG &) const override; - -/// Other utilities - const util::Duration & timeResolution() const override {return tstep_;} - const GeometryQG & resolution() const {return resol_;} - const oops::Variables & variables() const override {return linvars_;} - - private: - void print(std::ostream &) const override; - -// Data - F90model keyConfig_; - util::Duration tstep_; - const GeometryQG resol_; - oops::Variables linvars_; -}; -// ----------------------------------------------------------------------------- - -} // namespace qg -#endif // QG_MODEL_TLMIDQG_H_ diff --git a/qg/model/TlmQG.cc b/qg/model/TlmQG.cc index 389013d36..82031d815 100644 --- a/qg/model/TlmQG.cc +++ b/qg/model/TlmQG.cc @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -22,6 +22,7 @@ #include "model/GeometryQG.h" #include "model/IncrementQG.h" +#include "model/ModelBias.h" #include "model/ModelBiasIncrement.h" #include "model/ModelQG.h" #include "model/QgFortran.h" @@ -30,7 +31,7 @@ namespace qg { // ----------------------------------------------------------------------------- -static oops::LinearModelMaker makerQGTLM_("QgTLM"); +static oops::interface::LinearModelMaker makerQGTLM_("QgTLM"); // ----------------------------------------------------------------------------- TlmQG::TlmQG(const GeometryQG & resol, const eckit::Configuration & tlConf) : keyConfig_(0), tstep_(), resol_(resol), traj_(), diff --git a/qg/model/TlmQG.h b/qg/model/TlmQG.h index 856d34ddc..238b59c74 100644 --- a/qg/model/TlmQG.h +++ b/qg/model/TlmQG.h @@ -17,8 +17,8 @@ #include -#include "oops/base/LinearModelBase.h" #include "oops/base/Variables.h" +#include "oops/interface/LinearModelBase.h" #include "oops/util/Duration.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -39,7 +39,7 @@ namespace qg { * QG linear model definition and configuration parameters. */ -class TlmQG: public oops::LinearModelBase, +class TlmQG: public oops::interface::LinearModelBase, private util::ObjectCounter { public: static const std::string classname() {return "qg::TlmQG";} @@ -47,16 +47,17 @@ class TlmQG: public oops::LinearModelBase, TlmQG(const GeometryQG &, const eckit::Configuration &); ~TlmQG(); -/// Model trajectory computation - void setTrajectory(const StateQG &, StateQG &, const ModelBias &) override; - -/// Run TLM and its adjoint + /// Prepare model integration void initializeTL(IncrementQG &) const override; - void stepTL(IncrementQG &, const ModelBiasIncrement &) const override; - void finalizeTL(IncrementQG &) const override; - void initializeAD(IncrementQG &) const override; + + /// Model integration + void stepTL(IncrementQG &, const ModelBiasIncrement &) const override; void stepAD(IncrementQG &, ModelBiasIncrement &) const override; + void setTrajectory(const StateQG &, StateQG &, const ModelBias &) override; + + /// Finish model integration + void finalizeTL(IncrementQG &) const override; void finalizeAD(IncrementQG &) const override; /// Other utilities diff --git a/qg/model/instantiateQgLocalizationFactory.h b/qg/model/instantiateQgLocalizationFactory.h index 0540139ed..720d61dba 100644 --- a/qg/model/instantiateQgLocalizationFactory.h +++ b/qg/model/instantiateQgLocalizationFactory.h @@ -11,7 +11,7 @@ #ifndef QG_MODEL_INSTANTIATEQGLOCALIZATIONFACTORY_H_ #define QG_MODEL_INSTANTIATEQGLOCALIZATIONFACTORY_H_ -#include "oops/interface/Localization.h" +#include "oops/interface/LocalizationBase.h" #include "oops/qg/LocalizationMatrixQG.h" #include "oops/qg/QgTraits.h" @@ -19,8 +19,7 @@ namespace qg { void instantiateQgLocalizationFactory() { - static oops::LocalizationMaker > makerQG_("QG"); + static oops::interface::LocalizationMaker makerQG_("QG"); } } // namespace qg diff --git a/qg/model/qg_fields_interface.F90 b/qg/model/qg_fields_interface.F90 index 5b975ec31..7d8ad0c7d 100644 --- a/qg/model/qg_fields_interface.F90 +++ b/qg/model/qg_fields_interface.F90 @@ -715,7 +715,6 @@ subroutine qg_fields_serialize_c(c_key_fld,c_vsize,c_vect_fld) bind(c,name='qg_f ! Call Fortran call qg_fields_serialize(fld,c_vsize,c_vect_fld) - end subroutine qg_fields_serialize_c ! ------------------------------------------------------------------------------ !> Deserialize fields diff --git a/qg/model/qg_fields_mod.F90 b/qg/model/qg_fields_mod.F90 index 5fe06980a..724f61dfa 100644 --- a/qg/model/qg_fields_mod.F90 +++ b/qg/model/qg_fields_mod.F90 @@ -1145,19 +1145,16 @@ subroutine qg_fields_set_atlas(self,vars,afieldset) ! Get or create field do jvar=1,vars%nvars() fieldname = vars%variable(jvar) - if (afieldset%has_field(trim(fieldname))) then - ! Get afield - afield = afieldset%field(trim(fieldname)) - else + if (.not.afieldset%has_field(trim(fieldname))) then ! Create field afield = self%geom%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%geom%nz) ! Add field call afieldset%add(afield) - endif - ! Release pointer - call afield%final() + ! Release pointer + call afield%final() + endif enddo end subroutine qg_fields_set_atlas diff --git a/qg/model/qg_obsdb_mod.F90 b/qg/model/qg_obsdb_mod.F90 index ce4d587e5..7683b3cbe 100644 --- a/qg/model/qg_obsdb_mod.F90 +++ b/qg/model/qg_obsdb_mod.F90 @@ -394,13 +394,13 @@ subroutine qg_obsdb_read(self,winbgn,winend) type(datetime),intent(in) :: winend !< End of window ! Local variables -integer :: igrp,icol,iobs,ncol,nobsfile,jobs -integer :: ncid,grpname_id,ngrp_id,nobs_id,ncol_id,times_id,nlev_id,colname_id,values_id +integer,parameter :: ngrpmax = 3 +integer,parameter :: ncolmax = 7 +integer :: grp_ids(ngrpmax),igrp,nobs_in_grp,iobs,jobs,ncol,col_ids(ncolmax),icol,nlev_id,values_id +integer :: ncid,nobs_id,times_id type(group_data),pointer :: jgrp type(column_data),pointer :: jcol -character(len=6) :: igrpchar character(len=50) :: stime -character(len=1024) :: record logical, allocatable :: inwindow(:) type(datetime) :: tobs type(datetime), allocatable :: alltimes(:) @@ -409,14 +409,8 @@ subroutine qg_obsdb_read(self,winbgn,winend) ! Open NetCDF file call ncerr(nf90_open(trim(self%filein),nf90_nowrite,ncid)) -! Get dimensions ids -call ncerr(nf90_inq_dimid(ncid,'ngrp',ngrp_id)) - -! Get dimensions -call ncerr(nf90_inquire_dimension(ncid,ngrp_id,len=self%ngrp)) - -! Get variables ids -call ncerr(nf90_inq_varid(ncid,'grpname',grpname_id)) +! Get groups ids +call ncerr(nf90_inq_grps(ncid,self%ngrp,grp_ids)) do igrp=1,self%ngrp ! Allocation @@ -427,53 +421,51 @@ subroutine qg_obsdb_read(self,winbgn,winend) allocate(jgrp%next) jgrp => jgrp%next endif - write(igrpchar,'(i6.6)') igrp - ! Get variables - call ncerr(nf90_get_var(ncid,grpname_id,jgrp%grpname,(/1,igrp/),(/50,1/))) + ! Get group name + call ncerr(nf90_inq_grpname(grp_ids(igrp),jgrp%grpname)) - ! Get dimensions ids - call ncerr(nf90_inq_dimid(ncid,'nobs_'//igrpchar,nobs_id)) - call ncerr(nf90_inq_dimid(ncid,'ncol_'//igrpchar,ncol_id)) + ! Get dimension id + call ncerr(nf90_inq_dimid(grp_ids(igrp),'nobs',nobs_id)) - ! Get dimensions - call ncerr(nf90_inquire_dimension(ncid,nobs_id,len=nobsfile)) - call ncerr(nf90_inquire_dimension(ncid,ncol_id,len=ncol)) + ! Get dimension + call ncerr(nf90_inquire_dimension(grp_ids(igrp),nobs_id,len=nobs_in_grp)) - ! Get variables ids - call ncerr(nf90_inq_varid(ncid,'times_'//igrpchar,times_id)) - call ncerr(nf90_inq_varid(ncid,'nlev_'//igrpchar,nlev_id)) - call ncerr(nf90_inq_varid(ncid,'colname_'//igrpchar,colname_id)) - call ncerr(nf90_inq_varid(ncid,'values_'//igrpchar,values_id)) + ! Get variable id + call ncerr(nf90_inq_varid(grp_ids(igrp),'times',times_id)) ! Allocation - allocate(inwindow(nobsfile)) - allocate(alltimes(nobsfile)) + allocate(inwindow(nobs_in_grp)) + allocate(alltimes(nobs_in_grp)) ! Read in times - call ncerr(nf90_get_var(ncid,grpname_id,jgrp%grpname,(/1,igrp/),(/50,1/))) jgrp%nobs = 0 - do iobs=1,nobsfile - call ncerr(nf90_get_var(ncid,times_id,stime,(/1,iobs/),(/50,1/))) + do iobs=1,nobs_in_grp + call ncerr(nf90_get_var(grp_ids(igrp),times_id,stime,(/1,iobs/),(/50,1/))) call datetime_create(stime,tobs) - if (tobs > winbgn .and. tobs <= winend) then + if ((tobs > winbgn).and.(tobs <= winend)) then inwindow(iobs) = .true. alltimes(iobs) = tobs - jgrp%nobs = jgrp%nobs + 1 + jgrp%nobs = jgrp%nobs+1 else inwindow(iobs) = .false. endif - end do + enddo + ! Allocation allocate(jgrp%times(jgrp%nobs)) + + ! Copy times jobs=0 - do iobs=1,nobsfile + do iobs=1,nobs_in_grp if (inwindow(iobs)) then - jobs = jobs + 1 + jobs = jobs+1 jgrp%times(jobs) = alltimes(iobs) endif end do - deallocate(alltimes) + + ! Count columns + call ncerr(nf90_inq_grps(grp_ids(igrp),ncol,col_ids)) ! Loop over columns do icol=1,ncol @@ -486,26 +478,40 @@ subroutine qg_obsdb_read(self,winbgn,winend) jcol => jcol%next endif - ! Get variables - call ncerr(nf90_get_var(ncid,nlev_id,jcol%nlev,(/icol/))) - call ncerr(nf90_get_var(ncid,colname_id,jcol%colname,(/1,icol/),(/50,1/))) + ! Get column name + call ncerr(nf90_inq_grpname(col_ids(icol),jcol%colname)) + + ! Get dimension id + call ncerr(nf90_inq_dimid(col_ids(icol),'nlev',nlev_id)) + + ! Get dimension + call ncerr(nf90_inquire_dimension(col_ids(icol),nlev_id,len=jcol%nlev)) + + ! Get variable id + call ncerr(nf90_inq_varid(col_ids(icol),'values',values_id)) ! Allocation - allocate(readbuf(jcol%nlev,nobsfile)) + allocate(readbuf(jcol%nlev,nobs_in_grp)) allocate(jcol%values(jcol%nlev,jgrp%nobs)) ! Get values - call ncerr(nf90_get_var(ncid,values_id,readbuf(1:jcol%nlev,:),(/1,icol,1/),(/jcol%nlev,1,nobsfile/))) + call ncerr(nf90_get_var(col_ids(icol),values_id,readbuf(1:jcol%nlev,:),(/1,1/),(/jcol%nlev,nobs_in_grp/))) + ! Copy values jobs = 0 - do iobs=1,nobsfile + do iobs=1,nobs_in_grp if (inwindow(iobs)) then - jobs = jobs + 1 + jobs = jobs+1 jcol%values(:,jobs) = readbuf(:,iobs) endif enddo + + ! Release memory deallocate(readbuf) enddo + + ! Release memory + deallocate(alltimes) deallocate(inwindow) enddo @@ -523,82 +529,57 @@ subroutine qg_obsdb_write(self) type(qg_obsdb),intent(in) :: self !< Observation data ! Local variables -integer :: igrp,icol,iobs,ncol,nlevmax -integer :: ncid,nstrmax_id,grpname_id,ngrp_id,nobs_id,ncol_id,nlevmax_id,times_id,nlev_id,colname_id,values_id +integer :: iobs +integer :: ncid,nstrmax_id,grp_id,nobs_id,times_id,col_id,nlev_id,values_id type(group_data),pointer :: jgrp type(column_data),pointer :: jcol -character(len=6) :: igrpchar character(len=50) :: stime ! Create NetCDF file -call ncerr(nf90_create(trim(self%fileout),or(nf90_clobber,nf90_64bit_offset),ncid)) +call ncerr(nf90_create(trim(self%fileout),or(nf90_clobber,nf90_netcdf4),ncid)) ! Define dimensions call ncerr(nf90_def_dim(ncid,'nstrmax',50,nstrmax_id)) -call ncerr(nf90_def_dim(ncid,'ngrp',self%ngrp,ngrp_id)) - -! Define variable -call ncerr(nf90_def_var(ncid,'grpname',nf90_char,(/nstrmax_id,ngrp_id/),grpname_id)) - -! End definitions -call ncerr(nf90_enddef(ncid)) ! Loop over groups -igrp = 0 jgrp => self%grphead do while (associated(jgrp)) - igrp = igrp+1 if (jgrp%nobs > 0) then - write(igrpchar,'(i6.6)') igrp - ! Enter definitions mode - call ncerr(nf90_redef(ncid)) - - ! Compute dimensions - ncol = 0 - nlevmax = 0 - jcol => jgrp%colhead - do while (associated(jcol)) - ncol = ncol+1 - nlevmax = max(jcol%nlev,nlevmax) - jcol => jcol%next - enddo + ! Create group + call ncerr(nf90_def_grp(ncid,jgrp%grpname,grp_id)) - ! Define dimensions - call ncerr(nf90_def_dim(ncid,'nobs_'//igrpchar,jgrp%nobs,nobs_id)) - call ncerr(nf90_def_dim(ncid,'ncol_'//igrpchar,ncol,ncol_id)) - call ncerr(nf90_def_dim(ncid,'nlevmax_'//igrpchar,nlevmax,nlevmax_id)) + ! Define dimension + call ncerr(nf90_def_dim(grp_id,'nobs',jgrp%nobs,nobs_id)) ! Define variable - call ncerr(nf90_def_var(ncid,'times_'//igrpchar,nf90_char,(/nstrmax_id,nobs_id/),times_id)) - call ncerr(nf90_def_var(ncid,'nlev_'//igrpchar,nf90_int,(/ncol_id/),nlev_id)) - call ncerr(nf90_def_var(ncid,'colname_'//igrpchar,nf90_char,(/nstrmax_id,ncol_id/),colname_id)) - call ncerr(nf90_def_var(ncid,'values_'//igrpchar,nf90_double,(/nlevmax_id,ncol_id,nobs_id/),values_id)) - - ! End definitions - call ncerr(nf90_enddef(ncid)) + call ncerr(nf90_def_var(grp_id,'times',nf90_char,(/nstrmax_id,nobs_id/),times_id)) - ! Put variables - call ncerr(nf90_put_var(ncid,grpname_id,jgrp%grpname,(/1,igrp/),(/50,1/))) + ! Put variable do iobs=1,jgrp%nobs call datetime_to_string(jgrp%times(iobs),stime) - call ncerr(nf90_put_var(ncid,times_id,stime,(/1,iobs/),(/50,1/))) + call ncerr(nf90_put_var(grp_id,times_id,stime,(/1,iobs/),(/50,1/))) end do ! Loop over columns - icol = 0 jcol => jgrp%colhead do while (associated(jcol)) - icol = icol+1 + ! Create subgroup + call ncerr(nf90_def_grp(grp_id,jcol%colname,col_id)) - ! Put variables - call ncerr(nf90_put_var(ncid,nlev_id,jcol%nlev,(/icol/))) - call ncerr(nf90_put_var(ncid,colname_id,jcol%colname,(/1,icol/),(/50,1/))) - call ncerr(nf90_put_var(ncid,values_id,jcol%values(1:jcol%nlev,:),(/1,icol,1/),(/jcol%nlev,1,jgrp%nobs/))) + ! Define dimension + call ncerr(nf90_def_dim(col_id,'nlev',jcol%nlev,nlev_id)) + + ! Define variable + call ncerr(nf90_def_var(col_id,'values',nf90_double,(/nlev_id,nobs_id/),values_id)) + + ! Put variable + call ncerr(nf90_put_var(col_id,values_id,jcol%values(1:jcol%nlev,:),(/1,1/),(/jcol%nlev,jgrp%nobs/))) ! Update jcol => jcol%next enddo endif + ! Update jgrp=>jgrp%next end do diff --git a/qg/model/qg_obsvec_interface.F90 b/qg/model/qg_obsvec_interface.F90 index 53b19bd64..9fc014c24 100644 --- a/qg/model/qg_obsvec_interface.F90 +++ b/qg/model/qg_obsvec_interface.F90 @@ -126,14 +126,14 @@ subroutine qg_obsvec_zero_c(c_key_self) bind(c,name='qg_obsvec_zero_f90') end subroutine qg_obsvec_zero_c ! ------------------------------------------------------------------------------ -!> Set i-th value of the observation vector to zero -subroutine qg_obsvec_zero_ith_c(c_key_self, i) bind(c,name='qg_obsvec_zero_ith_f90') +!> Set i-th value of the observation vector to missing value +subroutine qg_obsvec_settomissing_ith_c(c_key_self, i) bind(c,name='qg_obsvec_settomissing_ith_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< Observation vector -integer(c_int),intent(in) :: i !< index of value to be set to zero +integer(c_int),intent(in) :: i !< index of value to be set to missing value ! Local variables type(qg_obsvec),pointer :: self @@ -142,9 +142,9 @@ subroutine qg_obsvec_zero_ith_c(c_key_self, i) bind(c,name='qg_obsvec_zero_ith_f ! Call Fortran ! increase index by 1 (C indices start with 0; Fortran indices start with 1) -call qg_obsvec_zero_ith(self, i+1) +call qg_obsvec_settomissing_ith(self, i+1) -end subroutine qg_obsvec_zero_ith_c +end subroutine qg_obsvec_settomissing_ith_c ! ------------------------------------------------------------------------------ !> Set observation vector to ones subroutine qg_obsvec_ones_c(c_key_self) bind(c,name='qg_obsvec_ones_f90') @@ -186,6 +186,27 @@ subroutine qg_obsvec_mask_c(c_key_self,c_key_mask) bind(c,name='qg_obsvec_mask_f end subroutine qg_obsvec_mask_c ! ------------------------------------------------------------------------------ +!> Mask self observation vector (set values to missing where mask is a missing value) +subroutine qg_obsvec_mask_with_missing_c(c_key_self,c_key_mask) bind(c,name='qg_obsvec_mask_with_missing_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_self !< Observation vector +integer(c_int),intent(in) :: c_key_mask !< Mask + +! Local variables +type(qg_obsvec),pointer :: self,mask + +! Interface +call qg_obsvec_registry%get(c_key_self,self) +call qg_obsvec_registry%get(c_key_mask,mask) + +! Call Fortran +call qg_obsvec_mask_with_missing(self,mask) + +end subroutine qg_obsvec_mask_with_missing_c +! ------------------------------------------------------------------------------ !> Multiply observation vector with a scalar subroutine qg_obsvec_mul_scal_c(c_key_self,zz) bind(c,name='qg_obsvec_mul_scal_f90') diff --git a/qg/model/qg_obsvec_mod.F90 b/qg/model/qg_obsvec_mod.F90 index f062e97f0..58cf8ba51 100644 --- a/qg/model/qg_obsvec_mod.F90 +++ b/qg/model/qg_obsvec_mod.F90 @@ -20,9 +20,9 @@ module qg_obsvec_mod public :: qg_obsvec public :: qg_obsvec_registry public :: qg_obsvec_setup,qg_obsvec_clone,qg_obsvec_delete,qg_obsvec_copy,qg_obsvec_zero, & - & qg_obsvec_zero_ith, qg_obsvec_ones, qg_obsvec_mask, qg_obsvec_mul_scal,qg_obsvec_add, & - & qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div,qg_obsvec_axpy,qg_obsvec_invert, & - & qg_obsvec_random,qg_obsvec_dotprod,qg_obsvec_stats, & + & qg_obsvec_settomissing_ith,qg_obsvec_ones,qg_obsvec_mask,qg_obsvec_mask_with_missing, & + & qg_obsvec_mul_scal,qg_obsvec_add,qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div, & + & qg_obsvec_axpy,qg_obsvec_invert,qg_obsvec_random,qg_obsvec_dotprod,qg_obsvec_stats, & & qg_obsvec_size,qg_obsvec_nobs,qg_obsvec_nobs_withmask,qg_obsvec_get_withmask ! ------------------------------------------------------------------------------ interface @@ -155,8 +155,8 @@ subroutine qg_obsvec_zero(self) end subroutine qg_obsvec_zero ! ------------------------------------------------------------------------------ -!> Set i-th value of observation vector to zero -subroutine qg_obsvec_zero_ith(self, i) +!> Set i-th value of observation vector to missing value +subroutine qg_obsvec_settomissing_ith(self, i) implicit none @@ -165,9 +165,9 @@ subroutine qg_obsvec_zero_ith(self, i) integer, intent(in) :: i ! Set observation vector to zero -self%values(:,i) = 0.0 +self%values(:,i) = self%missing -end subroutine qg_obsvec_zero_ith +end subroutine qg_obsvec_settomissing_ith ! ------------------------------------------------------------------------------ !> Set observation vector to ones subroutine qg_obsvec_ones(self) @@ -195,7 +195,20 @@ subroutine qg_obsvec_mask(self,mask) where(mask%values == 1) self%values = self%missing end subroutine qg_obsvec_mask +! ------------------------------------------------------------------------------ +!> Mask observation vector (set values to missing values where mask == missing value) +subroutine qg_obsvec_mask_with_missing(self,mask) +implicit none + +! Passed variables +type(qg_obsvec),intent(inout) :: self !< Observation vector +type(qg_obsvec),intent(in) :: mask !< mask + +if ((self%nobs/=mask%nobs).or.(self%nlev/=mask%nlev)) call abor1_ftn('qg_obsvec_mask: inconsistent sizes') + +where(mask%values == mask%missing) self%values = self%missing +end subroutine qg_obsvec_mask_with_missing ! ------------------------------------------------------------------------------ !> Multiply observation vector with a scalar subroutine qg_obsvec_mul_scal(self,zz) @@ -438,7 +451,8 @@ subroutine qg_obsvec_nobs_withmask(self,obsmask,kobs) integer,intent(inout) :: kobs !< Observation vector size ! Get observation vector size -kobs = count(mask = (self%values /= self%missing) .and. (obsmask%values == 0)) +kobs = count(mask = (self%values /= self%missing) .and. & + (obsmask%values /= obsmask%missing)) end subroutine qg_obsvec_nobs_withmask @@ -460,7 +474,8 @@ subroutine qg_obsvec_get_withmask(self,obsmask,vals,nvals) ! Loop over values do jobs=1,self%nobs do jlev=1,self%nlev - if ((self%values(jlev, jobs) /= self%missing) .and. (obsmask%values(jlev, jobs) == 0)) then + if ((self%values(jlev, jobs) /= self%missing) .and. & + (obsmask%values(jlev, jobs) /= obsmask%missing)) then if (jval > nvals) call abor1_ftn('qg_obsvec_get: inconsistent vector size') vals(jval) = self%values(jlev, jobs) jval = jval + 1 diff --git a/qg/model/qg_obsvec_random_f.cc b/qg/model/qg_obsvec_random_f.cc index 852568ac7..add03b812 100644 --- a/qg/model/qg_obsvec_random_f.cc +++ b/qg/model/qg_obsvec_random_f.cc @@ -14,7 +14,7 @@ namespace qg { // ----------------------------------------------------------------------------- void qg_obsvec_random_f(const ObsSpaceQG & odb, const int & nn, double * xx) { - static util::NormalDistribution dist(nn, 0.0, 1.0, odb.getSeed()); + util::NormalDistribution dist(nn, 0.0, 1.0, odb.getSeed()); for (int jj = 0; jj < nn; ++jj) xx[jj] = dist[jj]; } diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 879814737..900450ea3 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -2,6 +2,7 @@ list( APPEND qg_testinput testinput/3densvar.yaml testinput/3dvar.yaml testinput/3dvar_change_var.yaml + testinput/3dvar_full_inverse.yaml testinput/3dvar_hybrid.yaml testinput/3dvar_hybrid_wo_jb_evaluation.yaml testinput/3dfgat.yaml @@ -78,6 +79,7 @@ list( APPEND qg_testinput testinput/static_b_init.yaml testinput/truth.yaml testinput/verticallocev.yaml + testinput/verticallocev_io.yaml testinput/uniform_field_hybrid.yaml testinput/uniform_field_inflation.yaml ) @@ -238,6 +240,12 @@ ecbuild_add_test( TARGET test_qg_verticallocev LIBS qg TEST_DEPENDS test_qg_truth ) +ecbuild_add_test( TARGET test_qg_verticallocev_io + SOURCES executables/TestVerticalLocEV.cc + ARGS "testinput/verticallocev_io.yaml" + LIBS qg + TEST_DEPENDS test_qg_truth ) + ecbuild_add_test( TARGET test_qg_modelauxincrement SOURCES executables/TestModelAuxIncrement.cc ARGS "testinput/interfaces.yaml" @@ -575,6 +583,12 @@ ecbuild_add_test( TARGET test_qg_3dvar_change_var COMMAND qg_4dvar.x TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) +ecbuild_add_test( TARGET test_qg_3dvar_full_inverse + OMP 2 + ARGS testinput/3dvar_full_inverse.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) + ecbuild_add_test( TARGET test_qg_3dvar_hybrid OMP 2 ARGS testinput/3dvar_hybrid.yaml diff --git a/qg/test/testinput/3dfgat.yaml b/qg/test/testinput/3dfgat.yaml index d3703188f..0483351a3 100644 --- a/qg/test/testinput/3dfgat.yaml +++ b/qg/test/testinput/3dfgat.yaml @@ -53,7 +53,14 @@ cost function: obs type: WSpeed variational: minimizer: - algorithm: DRIPCG + algorithm: DRPLanczos + online diagnostics: + max eigenvectors: 5 + eigenvector: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dfgat.eigenvecs + type: krylov iterations: - ninner: 10 gradient norm reduction: 1.0e-10 @@ -62,11 +69,9 @@ variational: ny: 20 depths: [4500.0, 5500.0] linear model: - name: QgIdTLM - trajectory: - tstep: PT1H + name: Identity tstep: PT6H - tlm variables: ["x"] + increment variables: ["x"] variable change: Identity diagnostics: departures: ombg diff --git a/qg/test/testinput/3dvar.yaml b/qg/test/testinput/3dvar.yaml index d066f5e2c..a5c96b454 100644 --- a/qg/test/testinput/3dvar.yaml +++ b/qg/test/testinput/3dvar.yaml @@ -74,11 +74,11 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: 3dvar.iter1 - type: in + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar.iter1 + type: in - diagnostics: departures: ombg gradient norm reduction: 1.0e-10 @@ -90,12 +90,12 @@ variational: test: on online diagnostics: write increment: true - increment: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: 3dvar.iter2 - type: in - analysis variables: [x] + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar.iter2 + type: in + analysis variables: [x] final: diagnostics: departures: oman diff --git a/qg/test/testinput/3dvar_full_inverse.yaml b/qg/test/testinput/3dvar_full_inverse.yaml new file mode 100644 index 000000000..32758e5f3 --- /dev/null +++ b/qg/test/testinput/3dvar_full_inverse.yaml @@ -0,0 +1,111 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + analysis variables: [x] + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + full inverse: true + full inverse iterations: 20 + observations: + - obs error: + covariance model: diagonal + obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_full_inverse.obs3d.nc + obs type: Stream + get values: + interpolation type: default_1 + linear get values: + interpolation type: default_a + - obs error: + covariance model: diagonal + obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_full_inverse.obs3d.nc + obs type: Wind + get values: + interpolation type: default_2 + linear get values: + interpolation type: default_b + - obs error: + covariance model: diagonal + obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_full_inverse.obs3d.nc + obs type: WSpeed + get values: + interpolation type: default_3 + linear get values: + interpolation type: default_c +variational: + minimizer: + algorithm: DRIPCG + iterations: + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] + test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar_full_inverse.iter1 + type: in + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar_full_inverse.iter2 + type: in + analysis variables: [x] +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: 3dvar_full_inverse + frequency: PT6H + type: an + +test: + reference filename: testoutput/3dvar.test diff --git a/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml index 564c78253..3a7c9ef5d 100644 --- a/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml +++ b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml @@ -47,7 +47,7 @@ cost function: obsdatain: obsfile: Data/truth.obs3d.nc obsdataout: - obsfile: Data/3dvar_hybrid.obs3d.nc + obsfile: Data/3dvar_hybrid.obs3d.nojb_eval.nc obs type: Stream obs error: covariance model: diagonal @@ -57,7 +57,7 @@ cost function: obsdatain: obsfile: Data/truth.obs3d.nc obsdataout: - obsfile: Data/3dvar_hybrid.obs3d.nc + obsfile: Data/3dvar_hybrid.obs3d.nojb_eval.nc obs type: Wind obs error: covariance model: diagonal @@ -67,7 +67,7 @@ cost function: obsdatain: obsfile: Data/truth.obs3d.nc obsdataout: - obsfile: Data/3dvar_hybrid.obs3d.nc + obsfile: Data/3dvar_hybrid.obs3d.nojb_eval.nc obs type: WSpeed obs error: covariance model: diagonal @@ -102,7 +102,7 @@ final: departures: oman output: datadir: Data - exp: 3dvar + exp: 3dvar_nojb_eval frequency: PT6H type: an diff --git a/qg/test/testinput/4dvar_drplanczos.yaml b/qg/test/testinput/4dvar_drplanczos.yaml index 5462da35c..f32ab9a77 100644 --- a/qg/test/testinput/4dvar_drplanczos.yaml +++ b/qg/test/testinput/4dvar_drplanczos.yaml @@ -63,11 +63,11 @@ variational: maxpairs: 3 online diagnostics: write basis: true - krylov basis: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: 4dvar.drplanczos - type: krylov + krylov basis: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 4dvar.drplanczos + type: krylov iterations: - ninner: 10 gradient norm reduction: 1.0e-10 diff --git a/qg/test/testinput/eda_3dfgat_1.yaml b/qg/test/testinput/eda_3dfgat_1.yaml index 820ef2b1a..019471013 100644 --- a/qg/test/testinput/eda_3dfgat_1.yaml +++ b/qg/test/testinput/eda_3dfgat_1.yaml @@ -72,11 +72,10 @@ variational: departures: ombg gradient norm reduction: 1.0e-10 linear model: - trajectory: - tstep: PT1H tstep: PT6H variable change: Identity - name: QgIdTLM + name: Identity + increment variables: ["x"] ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dfgat_2.yaml b/qg/test/testinput/eda_3dfgat_2.yaml index e06aca56a..578ac8e46 100644 --- a/qg/test/testinput/eda_3dfgat_2.yaml +++ b/qg/test/testinput/eda_3dfgat_2.yaml @@ -64,11 +64,10 @@ variational: departures: ombg gradient norm reduction: 1.0e-10 linear model: - trajectory: - tstep: PT1H tstep: PT6H variable change: Identity - name: QgIdTLM + name: Identity + increment variables: ["x"] ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dfgat_3.yaml b/qg/test/testinput/eda_3dfgat_3.yaml index 7529ddb6f..843ed7f12 100644 --- a/qg/test/testinput/eda_3dfgat_3.yaml +++ b/qg/test/testinput/eda_3dfgat_3.yaml @@ -64,11 +64,10 @@ variational: departures: ombg gradient norm reduction: 1.0e-10 linear model: - trajectory: - tstep: PT1H tstep: PT6H variable change: Identity - name: QgIdTLM + name: Identity + increment variables: ["x"] ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dfgat_4.yaml b/qg/test/testinput/eda_3dfgat_4.yaml index ec66dfbb4..b2a152ee2 100644 --- a/qg/test/testinput/eda_3dfgat_4.yaml +++ b/qg/test/testinput/eda_3dfgat_4.yaml @@ -64,11 +64,10 @@ variational: departures: ombg gradient norm reduction: 1.0e-10 linear model: - trajectory: - tstep: PT1H tstep: PT6H variable change: Identity - name: QgIdTLM + name: Identity + increment variables: ["x"] ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/verticallocev.yaml b/qg/test/testinput/verticallocev.yaml index 4392ebcd1..7e745aee7 100644 --- a/qg/test/testinput/verticallocev.yaml +++ b/qg/test/testinput/verticallocev.yaml @@ -3,7 +3,7 @@ geometry: ny: 20 depths: [4500, 5500, 6500, 7500, 8500, 9500] # note: these are only for the test (specifying 6 levels), # and have not been scientifically validated -test date: 2009-12-31T00:00:00Z +test date: &date 2009-12-31T00:00:00Z increment test: date: 2009-12-31T00:00:00Z @@ -14,4 +14,11 @@ vertical localization: lengthscale: 3 lengthscale units: levels fraction of retained variance: 0.5 + write eigen vectors: true + list of eigen vectors to write: + datadir: Data + date: *date + exp: verticalEVs.%{member}% + type: an + expected neig: 2 diff --git a/qg/test/testinput/verticallocev_io.yaml b/qg/test/testinput/verticallocev_io.yaml new file mode 100644 index 000000000..12b0637cf --- /dev/null +++ b/qg/test/testinput/verticallocev_io.yaml @@ -0,0 +1,22 @@ +geometry: + nx: 40 + ny: 20 + depths: [4500, 5500, 6500, 7500, 8500, 9500] # note: these are only for the test (specifying 6 levels), + # and have not been scientifically validated +test date: &date 2009-12-31T00:00:00Z + +increment test: + date: 2009-12-31T00:00:00Z + +inc variables: [x] +vertical localization: + nlevels: 6 + lengthscale: 3 + lengthscale units: levels + fraction of retained variance: 0.5 + read eigen vectors: true + list of eigen vectors to read: + - filename: Data/verticalEVs.001.an.2009-12-31T00:00:00Z.nc + - filename: Data/verticalEVs.002.an.2009-12-31T00:00:00Z.nc + +expected neig: 2 diff --git a/qg/test/testoutput/3dfgat.test b/qg/test/testoutput/3dfgat.test index 87b3f7ff4..afb60e9ff 100644 --- a/qg/test/testoutput/3dfgat.test +++ b/qg/test/testoutput/3dfgat.test @@ -3,7 +3,17 @@ CostJo : Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+ CostJo : Nonlinear Jo(Wind) = 1038.28, nobs = 600, Jo/n = 1.73047, err = 6 CostJo : Nonlinear Jo(WSpeed) = 139.319, nobs = 300, Jo/n = 0.464395, err = 12 CostFunction: Nonlinear J = 1177.6 -DRIPCGMinimizer: reduction in residual norm = 0.186377 +Eigenvalue 1 : 177.38 +Norm eigenvector = 1.3618e+16 +Eigenvalue 2 : 160.727 +Norm eigenvector = 1.49979e+16 +Eigenvalue 3 : 108.802 +Norm eigenvector = 1.22906e+16 +Eigenvalue 4 : 94.006 +Norm eigenvector = 1.44667e+16 +Eigenvalue 5 : 84.3602 +Norm eigenvector = 1.14219e+16 +DRPLanczosMinimizer: reduction in residual norm = 0.0712486 CostFunction::addIncrement: Analysis: Valid time: 2010-01-01T09:00:00Z Resolution = 40, 20, 2 diff --git a/qg/test/testoutput/addincrement.test b/qg/test/testoutput/addincrement.test index 682399afd..aadfdfeb2 100644 --- a/qg/test/testoutput/addincrement.test +++ b/qg/test/testoutput/addincrement.test @@ -1,16 +1,16 @@ State: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction : Min= -4.4552e+08, Max= 1.0419e+08, RMS= 1.7642e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 + Streamfunction : Min= -1.7328e+07, Max= 5.7430e+07, RMS= 8.4654e+06 State plus increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.6263e+08, Max= 1.0660e+08, RMS= 1.7472e+08 + Streamfunction : Min= -4.6116e+08, Max= 1.0923e+08, RMS= 1.7475e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/addincrement_scaled.test b/qg/test/testoutput/addincrement_scaled.test index d880052bf..1cbe0fbf7 100644 --- a/qg/test/testoutput/addincrement_scaled.test +++ b/qg/test/testoutput/addincrement_scaled.test @@ -1,20 +1,20 @@ State: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction : Min= -4.4552e+08, Max= 1.0419e+08, RMS= 1.7642e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 + Streamfunction : Min= -1.7328e+07, Max= 5.7430e+07, RMS= 8.4654e+06 Scaled the increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -9.2140e+06, Max= 2.8742e+07, RMS= 4.2153e+06 + Streamfunction : Min= -8.6638e+06, Max= 2.8715e+07, RMS= 4.2327e+06 State plus increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.5411e+08, Max= 1.0285e+08, RMS= 1.7552e+08 + Streamfunction : Min= -4.5311e+08, Max= 1.0473e+08, RMS= 1.7554e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/convertincrement.test b/qg/test/testoutput/convertincrement.test index 131c580f4..68b6c4cfb 100644 --- a/qg/test/testoutput/convertincrement.test +++ b/qg/test/testoutput/convertincrement.test @@ -1,30 +1,30 @@ Input increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 + Streamfunction : Min= -1.7328e+07, Max= 5.7430e+07, RMS= 8.4654e+06 Trajectory state: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction : Min= -4.4552e+08, Max= 1.0419e+08, RMS= 1.7642e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Increment after variable transform: Valid time: 2010-01-01T12:00:00Z Resolution = 20, 10, 2 - Potential vorticity : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 + Potential vorticity : Min= -1.7458e-04, Max= 5.2972e-05, RMS= 2.0556e-05 Increment after variable transform: Valid time: 2010-01-01T12:00:00Z Resolution = 20, 10, 2 - Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 + Streamfunction : Min= -1.4229e+07, Max= 5.4972e+07, RMS= 7.4712e+06 Increment after variable transform: Valid time: 2010-01-01T12:00:00Z Resolution = 20, 10, 2 - Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 + Streamfunction : Min= -1.4229e+07, Max= 5.4972e+07, RMS= 7.4712e+06 Increment after variable transform: Valid time: 2010-01-01T12:00:00Z Resolution = 20, 10, 2 - Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 + Streamfunction : Min= -1.4229e+07, Max= 5.4972e+07, RMS= 7.4712e+06 Output increment: Valid time: 2010-01-01T12:00:00Z Resolution = 20, 10, 2 - Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 + Streamfunction : Min= -1.4229e+07, Max= 5.4972e+07, RMS= 7.4712e+06 diff --git a/qg/test/testoutput/diffstates.test b/qg/test/testoutput/diffstates.test index 6d813aba0..a1590bd4c 100644 --- a/qg/test/testoutput/diffstates.test +++ b/qg/test/testoutput/diffstates.test @@ -1,16 +1,16 @@ Input state 1: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction : Min= -4.4552e+08, Max= 1.0419e+08, RMS= 1.7642e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Input state 2: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.5934e+08, Max= 1.0390e+08, RMS= 1.7850e+08 + Streamfunction : Min= -4.5941e+08, Max= 1.0518e+08, RMS= 1.7848e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Output increment: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 + Streamfunction : Min= -1.7328e+07, Max= 5.7430e+07, RMS= 8.4654e+06 diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test index 33c42d827..cc3e7fea5 100644 --- a/qg/test/testoutput/eda_3dvar_block.test +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -3,51 +3,49 @@ CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329, err = 4e+ CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 CostFunction: Nonlinear J = 12705.8 - Norm reduction all members ( 1) = 0.402888, 0.342551, 0.430302, 0.347356 - Quadratic cost function all members: J ( 1) = 5681.52, 7492.09, 3160.43, 5694.31 - Norm reduction all members ( 2) = 0.1949, 0.165786, 0.260803, 0.171396 - Quadratic cost function all members: J ( 2) = 3979.77, 5093.07, 2507.38, 3641.38 - Norm reduction all members ( 3) = 0.158052, 0.116268, 0.185322, 0.0977361 - Quadratic cost function all members: J ( 3) = 3313.65, 4119.31, 2154.65, 2990.07 -DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 + Norm reduction all members ( 1) = 0.403356, 0.343225, 0.430509, 0.346888 + Quadratic cost function all members: J ( 1) = 5686.5, 7518.34, 3193.75, 5671.84 + Norm reduction all members ( 2) = 0.19501, 0.166099, 0.261324, 0.171379 + Quadratic cost function all members: J ( 2) = 3979.68, 5106.5, 2539.12, 3626.33 + Norm reduction all members ( 3) = 0.157963, 0.116257, 0.186766, 0.0975856 + Quadratic cost function all members: J ( 3) = 3313.54, 4131.01, 2182.39, 2977.22 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.157963 CostFunction::addIncrement: Analysis: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 + Streamfunction : Min= -4.4961e+08, Max= 1.0922e+08, RMS= 1.7739e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 CostJb : Nonlinear Jb = 31.4 CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err = 4e+06 CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 -CostJo : Nonlinear Jo(WSpeed) = 83.31, nobs = 300, Jo/n = 0.2777, err = 12 +CostJo : Nonlinear Jo(WSpeed) = 83.19, nobs = 300, Jo/n = 0.2773, err = 12 CostFunction: Nonlinear J = 3369 - Norm reduction all members ( 1) = 0.7216, 0.7109, 0.7893, 0.8465 - Quadratic cost function all members: J ( 1) = 3027, 3825, 2014, 2814 - Norm reduction all members ( 2) = 1.021, 0.9841, 1.035, 1.024 - Quadratic cost function all members: J ( 2) = 2506, 3217, 1761, 2438 - Norm reduction all members ( 3) = 0.7978, 0.7637, 0.8287, 0.8148 - Quadratic cost function all members: J ( 3) = 2136, 2681, 1581, 2157 - Norm reduction all members ( 4) = 0.426, 0.5472, 0.4854, 0.5317 - Quadratic cost function all members: J ( 4) = 1923, 2450, 1427, 1979 - Norm reduction all members ( 5) = 0.4426, 0.3654, 0.394, 0.4996 - Quadratic cost function all members: J ( 5) = 1784, 2130, 1326, 1798 - Norm reduction all members ( 6) = 0.3488, 0.2975, 0.3185, 0.3828 - Quadratic cost function all members: J ( 6) = 1652, 1980, 1247, 1648 - Norm reduction all members ( 7) = 0.2445, 0.2136, 0.2516, 0.3026 - Quadratic cost function all members: J ( 7) = 1550, 1898, 1189, 1576 - Norm reduction all members ( 8) = 0.1635, 0.1692, 0.1936, 0.2002 - Quadratic cost function all members: J ( 8) = 1447, 1789, 1136, 1491 - Norm reduction all members ( 9) = 0.1256, 0.1218, 0.1586, 0.1376 - Quadratic cost function all members: J ( 9) = 1403, 1740, 1113, 1437 -DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 + Norm reduction all members ( 1) = 0.7232, 0.7177, 0.7904, 0.8577 + Quadratic cost function all members: J ( 1) = 3027, 3842, 2040, 2804 + Norm reduction all members ( 2) = 1.023, 0.9856, 1.038, 1.039 + Quadratic cost function all members: J ( 2) = 2507, 3230, 1779, 2420 + Norm reduction all members ( 3) = 0.7852, 0.7523, 0.8272, 0.8239 + Quadratic cost function all members: J ( 3) = 2136, 2689, 1600, 2132 + Norm reduction all members ( 4) = 0.433, 0.5577, 0.4937, 0.5354 + Quadratic cost function all members: J ( 4) = 1925, 2458, 1443, 1953 + Norm reduction all members ( 5) = 0.4364, 0.366, 0.3961, 0.4956 + Quadratic cost function all members: J ( 5) = 1782, 2137, 1342, 1772 + Norm reduction all members ( 6) = 0.3469, 0.3007, 0.3383, 0.3857 + Quadratic cost function all members: J ( 6) = 1651, 1986, 1264, 1620 + Norm reduction all members ( 7) = 0.2414, 0.217, 0.2573, 0.2922 + Quadratic cost function all members: J ( 7) = 1549, 1903, 1208, 1545 + Norm reduction all members ( 8) = 0.1645, 0.1703, 0.1921, 0.1943 + Quadratic cost function all members: J ( 8) = 1448, 1791, 1148, 1459 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.1645 CostFunction::addIncrement: Analysis: Valid time: 2010-01-01T12:00:00Z Resolution = 40, 20, 2 - Streamfunction : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 + Streamfunction : Min= -4.6241e+08, Max= 1.0582e+08, RMS= 1.7688e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -CostJb : Nonlinear Jb = 162.6 -CostJo : Nonlinear Jo(Stream) = 563.7, nobs = 600, Jo/n = 0.9395, err = 4e+06 -CostJo : Nonlinear Jo(Wind) = 490.7, nobs = 600, Jo/n = 0.8179, err = 6 -CostJo : Nonlinear Jo(WSpeed) = 74.86, nobs = 300, Jo/n = 0.2495, err = 12 -CostFunction: Nonlinear J = 1292 +CostJb : Nonlinear Jb = 152.3 +CostJo : Nonlinear Jo(Stream) = 576.6, nobs = 600, Jo/n = 0.961, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 530.9, nobs = 600, Jo/n = 0.8849, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 79.87, nobs = 300, Jo/n = 0.2662, err = 12 +CostFunction: Nonlinear J = 1340 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2882aba5f..d74ffd69b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -43,7 +43,6 @@ oops/assimilation/HessianMatrix.h oops/assimilation/HMatrix.h oops/assimilation/HtMatrix.h oops/assimilation/HtRinvHMatrix.h -oops/assimilation/Increment4D.h oops/assimilation/IncrementalAssimilation.h oops/assimilation/instantiateCostFactory.h oops/assimilation/instantiateLocalEnsembleSolverFactory.h @@ -80,7 +79,6 @@ oops/assimilation/SaddlePointMinimizer.h oops/assimilation/SaddlePointPrecondMatrix.h oops/assimilation/SaddlePointVector.h oops/assimilation/SpectralLMP.h -oops/assimilation/State4D.h oops/assimilation/TriDiagSolve.h oops/assimilation/TriDiagSpectrum.h oops/assimilation/UpHessSolve.h @@ -94,7 +92,7 @@ oops/base/DolphChebyshev.cc oops/base/DolphChebyshev.h oops/base/EnsembleCovariance.h oops/base/GeneralizedDepartures.h -oops/base/GeoVaLsWriter.h +oops/base/Geometry.h oops/base/GetValuePost.h oops/base/GetValuePosts.h oops/base/GetValuesPost.h @@ -104,38 +102,40 @@ oops/base/LocalIncrement.cc oops/base/LocalIncrement.h oops/base/HybridCovariance.h oops/base/IdentityMatrix.h +oops/base/Increment.h +oops/base/Increment4D.h oops/base/IncrementEnsemble.h oops/base/IncrementEnsemble4D.h oops/base/InterpolatorBase.h oops/base/InterpolatorBase.cc oops/base/instantiateCovarFactory.h oops/base/instantiateObsFilterFactory.h -oops/base/LinearModelBase.h +oops/base/LinearModel.h oops/base/LinearVariableChangeBase.h oops/base/LinearVariableChangeParametersBase.h -oops/base/LocalizationBase.h -oops/base/ModelBase.h +oops/base/Localization.h +oops/base/Model.h oops/base/ModelSpaceCovarianceBase.h oops/base/ModelSpaceCovarianceParametersBase.h oops/base/ObsAuxControls.h oops/base/ObsAuxCovariances.h oops/base/ObsAuxIncrements.h oops/base/ObsEnsemble.h -oops/base/ObsErrorBase.h +oops/base/ObsError.h oops/base/ObsErrors.h oops/base/Observations.h oops/base/Observer.h oops/base/Observers.h oops/base/ObserversTLAD.h oops/base/ObserverTLAD.h -oops/base/ObsFilterBase.h -oops/base/ObsFilterParametersBase.h +oops/base/ObsFilter.h oops/base/ObsFilters.h oops/base/ObsLocalizationBase.h oops/base/ObsLocalizations.h oops/base/ObsSpaceBase.cc oops/base/ObsSpaceBase.h oops/base/ObsSpaces.h +oops/base/ObsVector.h oops/base/ParameterTraitsVariables.cc oops/base/ParameterTraitsVariables.h oops/base/PostBase.h @@ -145,6 +145,8 @@ oops/base/PostProcessorTLAD.h oops/base/PostTimer.cc oops/base/PostTimer.h oops/base/PostTimerParameters.h +oops/base/State.h +oops/base/State4D.h oops/base/StateEnsemble.h oops/base/StateEnsemble4D.h oops/base/StateInfo.h @@ -167,12 +169,14 @@ oops/contrib/dcmip_initial_conditions_test_4_v3.f90 oops/generic/gc99.h oops/generic/gc99.cc +oops/generic/GeoVaLsWriter.h +oops/generic/IdentityLinearModel.h oops/generic/IdentityModel.h oops/generic/IdLinearVariableChange.h oops/generic/IdVariableChange.h +oops/generic/instantiateLinearModelFactory.h oops/generic/instantiateModelFactory.h oops/generic/instantiateObsErrorFactory.h -oops/generic/instantiateTlmFactory.h oops/generic/instantiateVariableChangeFactory.h oops/generic/InterpolatorAtlas.h oops/generic/InterpolatorAtlas.cc @@ -180,9 +184,15 @@ oops/generic/interpolatorunstrc_f.h oops/generic/interpolatorunstrc_interface.F90 oops/generic/InterpolatorUnstructured.h oops/generic/InterpolatorUnstructured.cc -oops/generic/LinearModelId.h +oops/generic/LinearModelBase.h +oops/generic/LocalizationBase.h +oops/generic/ModelBase.h oops/generic/ObsErrorDiag.h +oops/generic/ObsErrorBase.h +oops/generic/ObsFilterBase.h +oops/generic/ObsFilterParametersBase.h oops/generic/PseudoModel.h +oops/generic/PseudoModelState4D.h oops/generic/soar.h oops/generic/soar.cc oops/generic/unstructured_interpolation_mod.F90 @@ -197,23 +207,23 @@ oops/interface/GeoVaLs.h oops/interface/GetValues.h oops/interface/Increment.h oops/interface/LinearGetValues.h -oops/interface/LinearModel.h +oops/interface/LinearModelBase.h oops/interface/LinearObsOperator.h oops/interface/LinearVariableChange.h -oops/interface/Localization.h +oops/interface/LocalizationBase.h oops/interface/Locations.h -oops/interface/Model.h oops/interface/ModelAuxControl.h oops/interface/ModelAuxCovariance.h oops/interface/ModelAuxIncrement.h +oops/interface/ModelBase.h oops/interface/ObsAuxControl.h oops/interface/ObsAuxCovariance.h oops/interface/ObsAuxIncrement.h oops/interface/ObsDataVector.h oops/interface/ObsDataVector_head.h oops/interface/ObsDiagnostics.h -oops/interface/ObsErrorCovariance.h -oops/interface/ObsFilter.h +oops/interface/ObsErrorBase.h +oops/interface/ObsFilterBase.h oops/interface/ObsLocalization.h oops/interface/ObsOperator.h oops/interface/ObsSpace.h @@ -250,6 +260,7 @@ oops/runs/Variational.h oops/util/abor1_cpp.cc oops/util/abor1_cpp.h oops/util/abor1_ftn.F90 +oops/util/algorithms.h oops/util/AnyOf.h oops/util/AssociativeContainers.h oops/util/CompareNVectors.h @@ -412,6 +423,7 @@ test/interface/VariableChange.h test/mpi/mpi.h test/generic/InterpolationInterface.h +test/generic/PseudoModelState4D.h test/generic/VerticalLocEV.h test/util/Fortran.h @@ -432,6 +444,7 @@ test/util/stringFunctions.h test/util/LocalEnvironment.h test/util/TestReference.h test/util/TypeTraits.h +test/util/algorithms.h ) list (APPEND oops_fheader_files @@ -478,7 +491,7 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/oops/util/linkedList_i.f target_sources( ${PROJECT_NAME} PRIVATE ${oops_test_src_files}) install( DIRECTORY test DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oops FILES_MATCHING PATTERN "*.h") -target_compile_features( ${PROJECT_NAME} PUBLIC cxx_std_11 ) +target_compile_features( ${PROJECT_NAME} PUBLIC cxx_std_14 ) if(OpenMP_FOUND) target_link_libraries( ${PROJECT_NAME} PUBLIC OpenMP::OpenMP_Fortran OpenMP::OpenMP_CXX ) endif() @@ -495,6 +508,12 @@ if ( GPTL_FOUND ) target_link_libraries( ${PROJECT_NAME} PUBLIC GPTL::GPTL ) endif() +# Stack traces on floating point exceptions +include( backtrace_deps ) + +target_link_libraries( ${PROJECT_NAME} PUBLIC ${OOPS_STACKTRACE_${OOPS_STACKTRACE_PROVIDER}_LIBS} ) +target_compile_definitions( ${PROJECT_NAME} PRIVATE ${OOPS_STACKTRACE_${OOPS_STACKTRACE_PROVIDER}_DEFS} ) + #Configure include directory layout for build-tree to match install-tree set(OOPS_BUILD_DIR_INCLUDE_PATH ${CMAKE_BINARY_DIR}/${PROJECT_NAME}/include) add_custom_target(oops_test_headers ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${OOPS_BUILD_DIR_INCLUDE_PATH}/oops @@ -537,6 +556,11 @@ ecbuild_add_test( TARGET test_base_posttimer ARGS "test/testinput/empty.yaml" LIBS oops eckit) +ecbuild_add_test( TARGET test_util_signal_trap + SOURCES test/util/signal_trap.cc + LIBS oops) +set_property( TEST test_util_signal_trap PROPERTY WILL_FAIL TRUE ) + ecbuild_add_test( TARGET test_util_random SOURCES test/base/Random.cc ARGS "test/testinput/random.yaml" @@ -630,6 +654,11 @@ ecbuild_add_test( TARGET test_oops_unstructured_interpolation ARGS "test/testinput/unstructured_interpolation.yaml" LIBS oops ) +ecbuild_add_test( TARGET test_util_algorithms + SOURCES test/util/algorithms.cc + ARGS "test/testinput/empty.yaml" + LIBS oops ) + ecbuild_add_test( TARGET test_util_comparenvectors SOURCES test/util/CompareNVectors.cc ARGS "test/testinput/empty.yaml" diff --git a/src/oops/assimilation/CalcHofX.h b/src/oops/assimilation/CalcHofX.h index 4f435c192..d1e3d97a6 100644 --- a/src/oops/assimilation/CalcHofX.h +++ b/src/oops/assimilation/CalcHofX.h @@ -17,13 +17,13 @@ #include "oops/base/Observations.h" #include "oops/base/ObsFilters.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/Locations.h" #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" @@ -35,8 +35,10 @@ template class CalcHofXParameters : public Parameters { OOPS_CONCRETE_PARAMETERS(CalcHofXParameters, Parameters) + typedef typename OBS::ObsOperator::Parameters_ ObsOperatorParameters_; + public: - oops::RequiredParameter obsOperator{"obs operator", this}; + oops::RequiredParameter obsOperator{"obs operator", this}; oops::Parameter>> obsFilters{"obs filters", {}, this}; }; @@ -99,7 +101,7 @@ class CalcHofX { const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) ObsOperatorVec_ obsops_; // Obs operators LocationsVec_ locations_; // locations - const ObsAuxCtrls_ * ybias_; // Obs bias + const ObsAuxCtrls_ * biascoeff_; // bias coefficients ObsDataVec_ qcflags_; // QC flags ObsVectorVec_ obserrs_; // Obs error variances (used in QC filters) ObsFiltersVec_ filters_; // QC filters @@ -112,7 +114,7 @@ template CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, const eckit::Configuration & config) : obsconfig_(config), obspaces_(obspaces), obsops_(), locations_(), - ybias_(nullptr), qcflags_(), obserrs_(), filters_(), + biascoeff_(nullptr), qcflags_(), obserrs_(), filters_(), geovars_(obspaces_.size()) { std::vector obsconfs = config.getSubConfigurations(); @@ -144,7 +146,7 @@ CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, template void CalcHofX::initialize(const ObsAuxCtrls_ & obsaux, const int iteration) { std::vector obsconfs = obsconfig_.getSubConfigurations(); - ybias_ = &obsaux; + biascoeff_ = &obsaux; filters_.clear(); for (size_t jj = 0; jj < obspaces_.size(); ++jj) { CalcHofXParameters observerParams; @@ -156,7 +158,7 @@ void CalcHofX::initialize(const ObsAuxCtrls_ & obsaux, const int iteration) /// Set up variables requested from the model geovars_[jj] += obsops_[jj]->requiredVars(); - geovars_[jj] += (*ybias_)[jj].requiredVars(); + geovars_[jj] += (*biascoeff_)[jj].requiredVars(); geovars_[jj] += filters_[jj]->requiredVars(); } Log::trace() << "CalcHofX::initialize done" << std::endl; @@ -169,17 +171,19 @@ Observations CalcHofX::compute(const GeoVaLsVec_ & geovals) { oops::Log::trace() << "CalcHofX::compute start" << std::endl; Observations yobs(obspaces_); + Observations ybias(obspaces_); + ybias.zero(); for (size_t jj = 0; jj < obspaces_.size(); ++jj) { /// call prior filters filters_[jj]->priorFilter(*geovals[jj]); /// compute H(x) oops::Variables vars; vars += filters_[jj]->requiredHdiagnostics(); - vars += (*ybias_)[jj].requiredHdiagnostics(); + vars += (*biascoeff_)[jj].requiredHdiagnostics(); ObsDiags_ ydiags(obspaces_[jj], *locations_[jj], vars); - obsops_[jj]->simulateObs(*geovals[jj], yobs[jj], (*ybias_)[jj], ydiags); + obsops_[jj]->simulateObs(*geovals[jj], yobs[jj], (*biascoeff_)[jj], ybias[jj], ydiags); /// call posterior filters - filters_[jj]->postFilter(yobs[jj], ydiags); + filters_[jj]->postFilter(yobs[jj], ybias[jj], ydiags); } oops::Log::trace() << "CalcHofX::compute done" << std::endl; return yobs; diff --git a/src/oops/assimilation/ControlIncrement.h b/src/oops/assimilation/ControlIncrement.h index d2a170608..400c434e8 100644 --- a/src/oops/assimilation/ControlIncrement.h +++ b/src/oops/assimilation/ControlIncrement.h @@ -18,9 +18,9 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/ObsAuxIncrements.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" #include "oops/interface/ModelAuxIncrement.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/ControlVariable.h b/src/oops/assimilation/ControlVariable.h index a1242cfc2..3a12f7a18 100644 --- a/src/oops/assimilation/ControlVariable.h +++ b/src/oops/assimilation/ControlVariable.h @@ -16,12 +16,12 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -134,8 +134,8 @@ void ControlVariable::write(const eckit::Configuration & config) con template void ControlVariable::print(std::ostream & outs) const { - outs << state_; - outs << modbias_; + outs << state_ << std::endl; + outs << modbias_ << std::endl; outs << obsbias_; } diff --git a/src/oops/assimilation/CostFct3DVar.h b/src/oops/assimilation/CostFct3DVar.h index 786f66618..ec2fab757 100644 --- a/src/oops/assimilation/CostFct3DVar.h +++ b/src/oops/assimilation/CostFct3DVar.h @@ -19,13 +19,13 @@ #include "oops/assimilation/CostJb3D.h" #include "oops/assimilation/CostJo.h" #include "oops/assimilation/CostTermBase.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/TrajectorySaver.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/CostFct4DEnsVar.h b/src/oops/assimilation/CostFct4DEnsVar.h index 1e7312060..d20b1feba 100644 --- a/src/oops/assimilation/CostFct4DEnsVar.h +++ b/src/oops/assimilation/CostFct4DEnsVar.h @@ -22,14 +22,14 @@ #include "oops/assimilation/CostJcDFI.h" #include "oops/assimilation/CostJo.h" #include "oops/assimilation/CostTermBase.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/TrajectorySaver.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/CostFct4DVar.h b/src/oops/assimilation/CostFct4DVar.h index 91254f44d..51a1069e1 100644 --- a/src/oops/assimilation/CostFct4DVar.h +++ b/src/oops/assimilation/CostFct4DVar.h @@ -21,17 +21,17 @@ #include "oops/assimilation/CostJcDFI.h" #include "oops/assimilation/CostJo.h" #include "oops/assimilation/CostTermBase.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/LinearModel.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/Model.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/TrajectorySaver.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/LinearModel.h" -#include "oops/interface/Model.h" -#include "oops/interface/State.h" #include "oops/interface/VariableChange.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/assimilation/CostFctWeak.h b/src/oops/assimilation/CostFctWeak.h index 3dd523820..b8c36f91c 100644 --- a/src/oops/assimilation/CostFctWeak.h +++ b/src/oops/assimilation/CostFctWeak.h @@ -24,17 +24,17 @@ #include "oops/assimilation/CostJcDFI.h" #include "oops/assimilation/CostJo.h" #include "oops/assimilation/CostTermBase.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/LinearModel.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/Model.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/TrajectorySaver.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/LinearModel.h" -#include "oops/interface/Model.h" -#include "oops/interface/State.h" #include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" diff --git a/src/oops/assimilation/CostFunction.h b/src/oops/assimilation/CostFunction.h index 843d70f36..59387d75e 100644 --- a/src/oops/assimilation/CostFunction.h +++ b/src/oops/assimilation/CostFunction.h @@ -28,12 +28,12 @@ #include "oops/assimilation/CostTermBase.h" #include "oops/assimilation/DualVector.h" #include "oops/assimilation/JqTermTLAD.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" diff --git a/src/oops/assimilation/CostJb3D.h b/src/oops/assimilation/CostJb3D.h index ab82daac0..6eaa72e18 100644 --- a/src/oops/assimilation/CostJb3D.h +++ b/src/oops/assimilation/CostJb3D.h @@ -15,11 +15,11 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CostJbState.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" #include "oops/util/Duration.h" diff --git a/src/oops/assimilation/CostJb4D.h b/src/oops/assimilation/CostJb4D.h index 3f17991c6..d21a7349c 100644 --- a/src/oops/assimilation/CostJb4D.h +++ b/src/oops/assimilation/CostJb4D.h @@ -16,11 +16,11 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CostJbState.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/CostJbJq.h b/src/oops/assimilation/CostJbJq.h index a66a54107..24e364ac3 100644 --- a/src/oops/assimilation/CostJbJq.h +++ b/src/oops/assimilation/CostJbJq.h @@ -17,11 +17,11 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CostJbState.h" #include "oops/assimilation/JqTermTLAD.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/dot_product.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/CostJbState.h b/src/oops/assimilation/CostJbState.h index f13214e1a..d9cd1bdd5 100644 --- a/src/oops/assimilation/CostJbState.h +++ b/src/oops/assimilation/CostJbState.h @@ -14,9 +14,6 @@ #include #include -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" - namespace util { class Duration; } @@ -24,6 +21,7 @@ namespace util { namespace oops { // Forward declaration + template class Geometry; template class Increment; template class State; template class JqTermTLAD; diff --git a/src/oops/assimilation/CostJbTotal.h b/src/oops/assimilation/CostJbTotal.h index 0827b51a7..0cb9450b5 100644 --- a/src/oops/assimilation/CostJbTotal.h +++ b/src/oops/assimilation/CostJbTotal.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -18,12 +18,12 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostJbState.h" +#include "oops/base/Geometry.h" #include "oops/base/ObsAuxCovariances.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Geometry.h" +#include "oops/base/State.h" #include "oops/interface/ModelAuxCovariance.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" namespace oops { diff --git a/src/oops/assimilation/CostJcDFI.h b/src/oops/assimilation/CostJcDFI.h index 70eae1e58..8e5cb4f96 100644 --- a/src/oops/assimilation/CostJcDFI.h +++ b/src/oops/assimilation/CostJcDFI.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -19,15 +19,15 @@ #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/DolphChebyshev.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/base/WeightedDiff.h" #include "oops/base/WeightedDiffTLAD.h" #include "oops/base/WeightingFct.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 218265b44..867f6aa04 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -23,6 +23,7 @@ #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" +#include "oops/base/Geometry.h" #include "oops/base/GetValuePosts.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" @@ -31,9 +32,7 @@ #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" @@ -58,7 +57,6 @@ template class CostJo : public CostTermBase Geometry_; typedef GetValuePosts GetValuePosts_; typedef State State_; - typedef Increment Increment_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; typedef Observers Observers_; @@ -115,7 +113,7 @@ template class CostJo : public CostTermBase yobs_; ObsErrors_ Rmat_; Observers_ observers_; @@ -136,7 +134,7 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi const util::DateTime & winbgn, const util::DateTime & winend, const eckit::mpi::Comm & ctime) : obsconf_(joConf), obspaces_(obsconf_, comm, winbgn, winend, ctime), - yobs_(obspaces_, "ObsValue"), Rmat_(obsconf_, obspaces_), observers_(obspaces_, joConf), + Rmat_(obsconf_, obspaces_), observers_(obspaces_, joConf), gradFG_(), obstlad_(), currentConf_() { Log::trace() << "CostJo::CostJo" << std::endl; @@ -159,9 +157,8 @@ void CostJo::setPostProc(const CtrlVar_ & xx, const eckit::Configura gradFG_.reset(); currentConf_.reset(new eckit::LocalConfiguration(conf)); - const int iterout = currentConf_->getInt("iteration"); - observers_.initialize(xx.state().geometry(), xx.obsVar(), Rmat_, pp, iterout); + observers_.initialize(xx.state().geometry(), xx.obsVar(), Rmat_, pp, conf); Log::trace() << "CostJo::setPostProc done" << std::endl; } @@ -173,18 +170,20 @@ double CostJo::computeCost() { Log::trace() << "CostJo::computeCost start" << std::endl; // Obs, simulated obs and departures (held here for nice prints and diagnostics) + if (!yobs_) + yobs_.reset(new Observations_(obspaces_, "ObsValue")); Observations_ yeqv(obspaces_); observers_.finalize(yeqv); // Perturb observations according to obs error statistics bool obspert = currentConf_->getBool("obs perturbations", false); if (obspert) { - yobs_.perturb(Rmat_); - Log::info() << "Perturbed observations: " << yobs_ << std::endl; + yobs_->perturb(Rmat_); + Log::info() << "Perturbed observations: " << *yobs_ << std::endl; } // Compute observations departures and save to output file - Departures_ ydep(yeqv - yobs_); + Departures_ ydep(yeqv - *yobs_); if (currentConf_->has("diagnostics.departures")) { const std::string depname = currentConf_->getString("diagnostics.departures"); ydep.save(depname); @@ -195,7 +194,7 @@ double CostJo::computeCost() { Rmat_.inverseMultiply(*gradFG_); // Print diagnostics - Log::info() << "Jo Observations:" << std::endl << yobs_ + Log::info() << "Jo Observations:" << std::endl << *yobs_ << "End Jo Observations" << std::endl; Log::info() << "Jo Observations Equivalent:" << std::endl << yeqv diff --git a/src/oops/assimilation/CostTermBase.h b/src/oops/assimilation/CostTermBase.h index 53463472f..fe225f0d2 100644 --- a/src/oops/assimilation/CostTermBase.h +++ b/src/oops/assimilation/CostTermBase.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -15,11 +15,10 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" +#include "oops/base/Geometry.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" namespace eckit { class Configuration; @@ -37,7 +36,6 @@ namespace oops { template class CostTermBase { typedef Geometry Geometry_; typedef State State_; - typedef Increment Increment_; typedef PostProcessor PostProc_; typedef PostProcessorTLAD PostProcTLAD_; diff --git a/src/oops/assimilation/DRMinimizer.h b/src/oops/assimilation/DRMinimizer.h index 1921d7ab4..58a63a550 100644 --- a/src/oops/assimilation/DRMinimizer.h +++ b/src/oops/assimilation/DRMinimizer.h @@ -11,11 +11,11 @@ #ifndef OOPS_ASSIMILATION_DRMINIMIZER_H_ #define OOPS_ASSIMILATION_DRMINIMIZER_H_ +#include #include #include #include - #include "eckit/config/Configuration.h" #include "oops/assimilation/BMatrix.h" #include "oops/assimilation/ControlIncrement.h" @@ -93,6 +93,12 @@ DRMinimizer::doMinimize(const eckit::Configuration & config) { const Bmat_ B(J_); const HtRinvH_ HtRinvH(J_, runOnlineAdjTest); +// Define minimisation starting point + // dx + CtrlInc_ * dx = new CtrlInc_(J_.jb()); + // dxh = B^{-1} dx + CtrlInc_ dxh(J_.jb()); + // Compute RHS (sum B^{-1} dx_{i}) + H^T R^{-1} d // dx_i = x_i - x_{i-1}; dx_1 = x_1 - x_b CtrlInc_ rhs(J_.jb()); @@ -110,11 +116,14 @@ DRMinimizer::doMinimize(const eckit::Configuration & config) { rhs *= -1.0; Log::info() << classname() << " rhs" << rhs << std::endl; -// Define minimisation starting point - // dx - CtrlInc_ * dx = new CtrlInc_(J_.jb()); - // dxh = B^{-1} dx - CtrlInc_ dxh(J_.jb()); +// Check for zero gradient (for example if no obs) + const double gnorm = dot_product(rhs, rhs); + const double epsilon = config.getDouble("epsilon", std::numeric_limits::epsilon()); + Log::info() << "Initial RHS squared norm = " << gnorm << std::endl; + if (gnorm < epsilon) { + Log::info() << "RHS smaller than " << epsilon << ", returning." << std::endl; + return dx; + } // Set J[0] = 0.5 (x_i - x_b)^T B^{-1} (x_i - x_b) + 0.5 d^T R^{-1} d const double costJ0Jb = costJ0Jb_; diff --git a/src/oops/assimilation/DRPLanczosMinimizer.h b/src/oops/assimilation/DRPLanczosMinimizer.h index 0ec0cca65..b8c735dab 100644 --- a/src/oops/assimilation/DRPLanczosMinimizer.h +++ b/src/oops/assimilation/DRPLanczosMinimizer.h @@ -11,6 +11,8 @@ #ifndef OOPS_ASSIMILATION_DRPLANCZOSMINIMIZER_H_ #define OOPS_ASSIMILATION_DRPLANCZOSMINIMIZER_H_ +#include + #include #include #include @@ -96,7 +98,6 @@ template class DRPLanczosMinimizer : public DRMini // For diagnostics eckit::LocalConfiguration diagConf_; - int outerLoop_; }; // ============================================================================= @@ -105,7 +106,7 @@ template DRPLanczosMinimizer::DRPLanczosMinimizer(const eckit::Configuration & conf, const CostFct_ & J) : DRMinimizer(J), lmp_(conf), hvecs_(), vvecs_(), zvecs_(), alphas_(), - betas_(), diagConf_(conf), outerLoop_(0) {} + betas_(), diagConf_(conf) {} // ----------------------------------------------------------------------------- @@ -247,10 +248,11 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr for (unsigned int jj = 0; jj < ss.size(); ++jj) { dx.axpy(ss[jj], *zvecs_[jj]); dxh.axpy(ss[jj], *hvecs_[jj]); - if (outerLoop_ == 0) writeKrylovBasis(diagConf_, *zvecs_[jj], jj); } - ++outerLoop_; + // Compute and save the eigenvectors + writeEigenvectors(diagConf_, alphas_, betas_, dd, zvecs_, hvecs_, HtRinvH, pr, vv, zz); + util::printRunStats("DRPLanczos end"); return normReduction; } diff --git a/src/oops/assimilation/DualMinimizer.h b/src/oops/assimilation/DualMinimizer.h index 9ce0d3e8c..e0dffc9e6 100644 --- a/src/oops/assimilation/DualMinimizer.h +++ b/src/oops/assimilation/DualMinimizer.h @@ -11,10 +11,10 @@ #ifndef OOPS_ASSIMILATION_DUALMINIMIZER_H_ #define OOPS_ASSIMILATION_DUALMINIMIZER_H_ +#include #include #include - #include "eckit/config/Configuration.h" #include "oops/assimilation/BMatrix.h" #include "oops/assimilation/ControlIncrement.h" @@ -87,6 +87,8 @@ DualMinimizer::doMinimize(const eckit::Configuration & config) { const HMatrix H(J_); const HtMatrix Ht(J_); + CtrlInc_ * dx = new CtrlInc_(J_.jb()); + // Define minimisation starting point in dual space Dual_ vv; for (unsigned jj = 0; jj < J_.nterms(); ++jj) { @@ -101,6 +103,15 @@ DualMinimizer::doMinimize(const eckit::Configuration & config) { } rr *= -1.0; +// Check for zero gradient (for example if no obs) + const double rnorm = dot_product(rr, rr); + const double epsilon = config.getDouble("epsilon", std::numeric_limits::epsilon()); + Log::info() << "Initial RHS squared norm = " << rnorm << std::endl; + if (rnorm < epsilon) { + Log::info() << "RHS smaller than " << epsilon << ", returning." << std::endl; + return dx; + } + // Update rr if initial dx in model space is not a zero vector // rr = rr - Rinv H dx0 @@ -127,7 +138,6 @@ DualMinimizer::doMinimize(const eckit::Configuration & config) { Ht.multiply(vv, dh); dh.axpy(-vvp, g0); - CtrlInc_ * dx = new CtrlInc_(J_.jb()); B.multiply(dh, *dx); // BHtaug vvaug Log::info() << classname() << ": Estimated Final Jb = " diff --git a/src/oops/assimilation/DualVector.h b/src/oops/assimilation/DualVector.h index 5febe488f..5bb693f7b 100644 --- a/src/oops/assimilation/DualVector.h +++ b/src/oops/assimilation/DualVector.h @@ -19,7 +19,7 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/base/Departures.h" #include "oops/base/GeneralizedDepartures.h" -#include "oops/interface/Increment.h" +#include "oops/base/Increment.h" #include "oops/util/dot_product.h" namespace oops { diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 48d8142b6..10e3a1be3 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -12,28 +12,25 @@ #include #include #include +#include #include #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/gletkfInterface.h" #include "oops/assimilation/LETKFSolverParameters.h" #include "oops/assimilation/LocalEnsembleSolver.h" -#include "oops/assimilation/State4D.h" #include "oops/base/Departures.h" #include "oops/base/DeparturesEnsemble.h" -#include "oops/base/GetValuesPost.h" +#include "oops/base/Geometry.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsEnsemble.h" -#include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" -#include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State4D.h" #include "oops/base/StateEnsemble4D.h" #include "oops/generic/VerticalLocEV.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/ObsDataVector.h" #include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -54,13 +51,9 @@ class GETKFSolver : public LocalEnsembleSolver { typedef DeparturesEnsemble DeparturesEnsemble_; typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; - typedef GetValuesPost GetValuesPost_; typedef IncrementEnsemble4D IncrementEnsemble4D_; - typedef ObsDataVector ObsDataVector_; typedef ObsEnsemble ObsEnsemble_; - typedef ObsErrors ObsErrors_; typedef Observations Observations_; - typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; typedef State4D State4D_; typedef StateEnsemble4D StateEnsemble4D_; @@ -71,7 +64,8 @@ class GETKFSolver : public LocalEnsembleSolver { /// Constructor (allocates Wa, wa, HZb_, /// saves options from the config, computes VerticalLocEV_) - GETKFSolver(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t); + GETKFSolver(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t, + const State4D_ &); Observations_ computeHofX(const StateEnsemble4D_ &, size_t, bool) override; @@ -112,10 +106,11 @@ class GETKFSolver : public LocalEnsembleSolver { template GETKFSolver::GETKFSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config, size_t nens) - : LocalEnsembleSolver(obspaces, geometry, config, nens), + const eckit::Configuration & config, size_t nens, + const State4D_ & xbmean) + : LocalEnsembleSolver(obspaces, geometry, config, nens, xbmean), nens_(nens), geometry_(geometry), - vertloc_(geometry_, config.getSubConfiguration("local ensemble DA.vertical localization")), + vertloc_(config.getSubConfiguration("local ensemble DA.vertical localization"), xbmean[0]), neig_(vertloc_.neig()), nanal_(neig_*nens_), HZb_(obspaces, nanal_) { options_.deserialize(config); @@ -183,18 +178,12 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & for (size_t ieig = 0; ieig < neig_; ++ieig) { State4D_ tmpState = xx_mean; tmpState += Ztmp[ieig]; - - this->hofx_.resetQc(); - this->hofx_.initialize(this->obsaux_, iteration); - - std::vector getValuesConfig = - util::vectoriseAndFilter(this->obsconf_, "get values"); - - GetValuesPost_ getvals(this->obspaces_, this->hofx_.locations(), - this->hofx_.requiredVars(), getValuesConfig); - getvals.fill(tmpState); - // compute H(x) on filled in geovals and run the filters - Observations_ tmpObs = this->hofx_.compute(getvals.geovals()); + Observations_ tmpObs(this->obspaces_); + eckit::LocalConfiguration config; + config.set("save hofx", false); + config.set("save qc", false); + config.set("save obs errors", false); + this->computeHofX4D(config, tmpState, tmpObs); HZb_[ii] = tmpObs - yb_mean; tmpObs.save("hofxm"+std::to_string(iteration)+"_"+std::to_string(ieig+1)+ "_"+std::to_string(iens+1)); @@ -204,7 +193,6 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & } } } - this->hofx_.readQcFlags("EffectiveQC"); return yb_mean; } @@ -228,8 +216,8 @@ void GETKFSolver::computeWeights(const Eigen::VectorXd & dy, Eigen::MatrixXf YbOrig_f = YbOrig.cast(); Eigen::VectorXf R_invvar_f = R_invvar.cast(); - Eigen::MatrixXf Wa_f(this->nanal_, this->nens_); - Eigen::VectorXf wa_f(this->nanal_); + Eigen::MatrixXf Wa_f(nanal_, this->nens_); + Eigen::VectorXf wa_f(nanal_); // call into GSI interface to compute Wa and wa const int getkf_inflation = 0; @@ -330,15 +318,10 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg // create the local subset of observations Departures_ locvector(this->obspaces_); - std::vector> outside; - for (size_t jj = 0; jj < this->obspaces_.size(); ++jj) { - outside.push_back(std::make_shared(this->obspaces_[jj], - this->obspaces_[jj].obsvariables())); - } locvector.ones(); - this->obsloc_.computeLocalization(i, outside, locvector); - locvector.mask(this->hofx_.qcflags()); - Eigen::VectorXd local_omb_vec = this->omb_.packEigen(outside); + this->obsloc().computeLocalization(i, locvector); + locvector.mask(*(this->invVarR_)); + Eigen::VectorXd local_omb_vec = this->omb_.packEigen(locvector); if (local_omb_vec.size() == 0) { // no obs. so no need to update Wa_ and wa_ @@ -347,12 +330,12 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg } else { // if obs are present do normal KF update // get local Yb & HZ - Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(outside); - Eigen::MatrixXd local_HZ_mat = this->HZb_.packEigen(outside); + Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(locvector); + Eigen::MatrixXd local_HZ_mat = this->HZb_.packEigen(locvector); // create local obs errors - Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(outside); + Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(locvector); // and apply localization - Eigen::VectorXd localization = locvector.packEigen(outside); + Eigen::VectorXd localization = locvector.packEigen(locvector); local_invVarR_vec.array() *= localization.array(); computeWeights(local_omb_vec, local_HZ_mat, local_Yb_mat, local_invVarR_vec); applyWeights(bkg_pert, ana_pert, i); diff --git a/src/oops/assimilation/HMatrix.h b/src/oops/assimilation/HMatrix.h index ddcf6ce38..ef3ce0382 100644 --- a/src/oops/assimilation/HMatrix.h +++ b/src/oops/assimilation/HMatrix.h @@ -19,9 +19,9 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DualVector.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Increment.h" namespace oops { diff --git a/src/oops/assimilation/HtMatrix.h b/src/oops/assimilation/HtMatrix.h index e31b7bc11..07c74b3b9 100644 --- a/src/oops/assimilation/HtMatrix.h +++ b/src/oops/assimilation/HtMatrix.h @@ -16,9 +16,9 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DualVector.h" +#include "oops/base/Increment.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Increment.h" namespace oops { diff --git a/src/oops/assimilation/IncrementalAssimilation.h b/src/oops/assimilation/IncrementalAssimilation.h index 614923c91..ae332e0fd 100644 --- a/src/oops/assimilation/IncrementalAssimilation.h +++ b/src/oops/assimilation/IncrementalAssimilation.h @@ -20,8 +20,8 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/Minimizer.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" #include "oops/util/printRunStats.h" diff --git a/src/oops/assimilation/JqTermTLAD.h b/src/oops/assimilation/JqTermTLAD.h index 11d565216..f7a69578f 100644 --- a/src/oops/assimilation/JqTermTLAD.h +++ b/src/oops/assimilation/JqTermTLAD.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -14,9 +14,9 @@ #include #include +#include "oops/base/Increment.h" #include "oops/base/PostBaseTLAD.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/assimilation/LETKFSolver.h b/src/oops/assimilation/LETKFSolver.h index 3b0d6fe5f..1b3cfbc22 100644 --- a/src/oops/assimilation/LETKFSolver.h +++ b/src/oops/assimilation/LETKFSolver.h @@ -20,14 +20,13 @@ #include "oops/assimilation/LocalEnsembleSolver.h" #include "oops/base/Departures.h" #include "oops/base/DeparturesEnsemble.h" +#include "oops/base/Geometry.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsErrors.h" #include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/ObsDataVector.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -52,14 +51,15 @@ class LETKFSolver : public LocalEnsembleSolver { typedef GeometryIterator GeometryIterator_; typedef IncrementEnsemble4D IncrementEnsemble4D_; typedef ObsErrors ObsErrors_; - typedef ObsDataVector ObsDataVector_; typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; + typedef State4D State4D_; public: static const std::string classname() {return "oops::LETKFSolver";} - LETKFSolver(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t); + LETKFSolver(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t, + const State4D_ &); /// KF update + posterior inflation at a grid point location (GeometryIterator_) void measurementUpdate(const IncrementEnsemble4D_ &, @@ -93,8 +93,10 @@ class LETKFSolver : public LocalEnsembleSolver { template LETKFSolver::LETKFSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config, size_t nens) - : LocalEnsembleSolver(obspaces, geometry, config, nens), nens_(nens) + const eckit::Configuration & config, size_t nens, + const State4D_ & xbmean) + : LocalEnsembleSolver(obspaces, geometry, config, nens, xbmean), + nens_(nens) { options_.deserialize(config); const LETKFInflationParameters & inflopt = options_.infl; @@ -139,15 +141,10 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg // create the local subset of observations Departures_ locvector(this->obspaces_); - std::vector> outside; - for (size_t jj = 0; jj < this->obspaces_.size(); ++jj) { - outside.push_back(std::make_shared(this->obspaces_[jj], - this->obspaces_[jj].obsvariables())); - } locvector.ones(); - this->obsloc_.computeLocalization(i, outside, locvector); - locvector.mask(this->hofx_.qcflags()); - Eigen::VectorXd local_omb_vec = this->omb_.packEigen(outside); + this->obsloc().computeLocalization(i, locvector); + locvector.mask(*(this->invVarR_)); + Eigen::VectorXd local_omb_vec = this->omb_.packEigen(locvector); if (local_omb_vec.size() == 0) { // no obs. so no need to update Wa_ and wa_ @@ -156,11 +153,11 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg } else { // if obs are present do normal KF update // create local Yb - Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(outside); + Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(locvector); // create local obs errors - Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(outside); + Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(locvector); // and apply localization - Eigen::VectorXd localization = locvector.packEigen(outside); + Eigen::VectorXd localization = locvector.packEigen(locvector); local_invVarR_vec.array() *= localization.array(); computeWeights(local_omb_vec, local_Yb_mat, local_invVarR_vec); applyWeights(bkg_pert, ana_pert, i); diff --git a/src/oops/assimilation/LETKFSolverGSI.h b/src/oops/assimilation/LETKFSolverGSI.h index c18d574df..a76a4285f 100644 --- a/src/oops/assimilation/LETKFSolverGSI.h +++ b/src/oops/assimilation/LETKFSolverGSI.h @@ -30,8 +30,10 @@ class LETKFSolverGSI : public LETKFSolver { typedef Geometry Geometry_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; + typedef State4D State4D_; public: - LETKFSolverGSI(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t); + LETKFSolverGSI(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t, + const State4D_ &); /// Computes weights for ensemble update with local observations /// \param[in] omb Observation departures (nlocalobs) @@ -45,8 +47,9 @@ class LETKFSolverGSI : public LETKFSolver { template LETKFSolverGSI::LETKFSolverGSI(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config, size_t nens) - : LETKFSolver(obspaces, geometry, config, nens) + const eckit::Configuration & config, size_t nens, + const State4D_ & xbmean) + : LETKFSolver(obspaces, geometry, config, nens, xbmean) { } diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index 3e8532bef..dd21dd8dc 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -11,24 +11,28 @@ #include #include #include +#include #include #include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/CalcHofX.h" #include "oops/base/Departures.h" #include "oops/base/DeparturesEnsemble.h" -#include "oops/base/GetValuesPost.h" +#include "oops/base/Geometry.h" #include "oops/base/IncrementEnsemble4D.h" +#include "oops/base/Model.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsEnsemble.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/Observers.h" #include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State.h" #include "oops/base/StateEnsemble4D.h" -#include "oops/interface/Geometry.h" +#include "oops/generic/PseudoModelState4D.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/ModelAuxControl.h" #include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -38,27 +42,33 @@ namespace oops { /// \brief Base class for LETKF-type solvers template class LocalEnsembleSolver { - typedef CalcHofX CalcHofX_; + typedef Observers Observers_; typedef Departures Departures_; typedef DeparturesEnsemble DeparturesEnsemble_; typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; - typedef GetValuesPost GetValuesPost_; typedef IncrementEnsemble4D IncrementEnsemble4D_; - typedef ObsAuxControls ObsAuxControls_; + typedef ObsAuxControls ObsAux_; typedef ObsEnsemble ObsEnsemble_; typedef ObsErrors ObsErrors_; typedef Observations Observations_; typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; + typedef State4D State4D_; typedef StateEnsemble4D StateEnsemble4D_; + typedef PseudoModelState4D PseudoModel_; + typedef State State_; + typedef Model Model_; + typedef ModelAuxControl ModelAux_; + typedef ObsDataVector ObsData_; + typedef std::vector> ObsDataVec_; public: static const std::string classname() {return "oops::LocalEnsembleSolver";} /// initialize solver with \p obspaces, \p geometry, full \p config and \p nens ensemble size LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config, size_t nens); + const eckit::Configuration & config, size_t nens, const State4D_ & xbmean); virtual ~LocalEnsembleSolver() = default; /// computes ensemble H(\p xx), returns mean H(\p xx), saves as hofx \p iteration @@ -73,15 +83,24 @@ class LocalEnsembleSolver { virtual void copyLocalIncrement(const IncrementEnsemble4D_ & bg, const GeometryIterator_ & i, IncrementEnsemble4D_ & an) const; + /// compute H(x) based on 4D state \p xx and put the result into \p yy. Also sets up + /// R_ based on the QC filters run during H(x) + void computeHofX4D(const eckit::Configuration &, const State4D_ &, Observations_ &); + /// accessor to obs localizations + const ObsLocalizations_ & obsloc() const {return obsloc_;} + protected: + const Geometry_ & geometry_; ///< Geometry associated with the updated states + const ObsSpaces_ & obspaces_; ///< ObsSpaces used in the update + Departures_ omb_; ///< obs - mean(H(x)); set in computeHofX method + DeparturesEnsemble_ Yb_; ///< ensemble perturbations in the observation space; + /// set in computeHofX method + std::unique_ptr R_; ///< observation errors, set in computeHofX method + std::unique_ptr invVarR_; ///< inverse observation error variance; set in + /// computeHofX method + + private: const eckit::LocalConfiguration obsconf_; // configuration for observations - const ObsSpaces_ & obspaces_; // ObsSpaces - const ObsAuxControls_ obsaux_; // Obs bias - CalcHofX_ hofx_; // observer - Departures_ omb_; // obs - mean(H(x)) - DeparturesEnsemble_ Yb_; // ensemble perturbations in the observation space - std::unique_ptr R_; ///< observation errors - std::unique_ptr invVarR_; ///< inverse observation error variance ObsLocalizations_ obsloc_; ///< observation space localization }; @@ -90,11 +109,40 @@ class LocalEnsembleSolver { template LocalEnsembleSolver::LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config, size_t nens) - : obsconf_(config, "observations"), obspaces_(obspaces), obsaux_(obspaces_, obsconf_), - hofx_(obspaces, obsconf_), omb_(obspaces_), Yb_(obspaces_, nens), - obsloc_(obsconf_, obspaces_) -{ + const eckit::Configuration & config, size_t nens, + const State4D_ & xbmean) + : geometry_(geometry), obspaces_(obspaces), omb_(obspaces_), Yb_(obspaces_, nens), + obsconf_(config, "observations"), obsloc_(obsconf_, obspaces_) {} + +// ----------------------------------------------------------------------------- + +template +void LocalEnsembleSolver::computeHofX4D(const eckit::Configuration & config, + const State4D_ & xx, Observations_ & yy) { + // compute forecast length from State4D times + const std::vector times = xx.validTimes(); + const util::Duration flength = times[times.size()-1] - times[0]; + // observation window is passed to PseudoModel as the default model time step. + // This is required when State4D has a single state: in this case if the default + // model time step is not specified, it will be set to zero and no observations + // will be processed during the "forecast" + const util::Duration winlen = obspaces_.windowEnd() - obspaces_.windowStart(); + + // Setup PseudoModelState4D + std::unique_ptr pseudomodel(new PseudoModel_(xx, winlen)); + const Model_ model(std::move(pseudomodel)); + // Setup model and obs biases; obs errors + ModelAux_ moderr(geometry_, eckit::LocalConfiguration()); + ObsAux_ obsaux(obspaces_, obsconf_); + R_.reset(new ObsErrors_(obsconf_, obspaces_)); + // Setup and run the model forecast with observers + State_ init_xx = xx[0]; + PostProcessor post; + Observers_ hofx(obspaces_, obsconf_); + + hofx.initialize(geometry_, obsaux, *R_, post, config); + model.forecast(init_xx, moderr, flength, post); + hofx.finalize(yy); } // ----------------------------------------------------------------------------- @@ -116,48 +164,61 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb obsens[jj].read("hofx"+std::to_string(iteration)+"_"+std::to_string(jj+1)); Log::test() << "H(x) for member " << jj+1 << ":" << std::endl << obsens[jj] << std::endl; } - hofx_.readQcFlags("EffectiveQC"); + R_.reset(new ObsErrors_(obsconf_, obspaces_)); } else { // compute and save H(x) Log::debug() << "Computing H(X) online" << std::endl; + + // save QC filters and ob errors to be used for all other members + // do not save H(X) (saved explicitly below) + eckit::LocalConfiguration config; + + // save hofx means that hofx will be written out into ObsSpace; + // if run computeHofX4D several times with save hofx on, + // the hofx will be overwritten, + // unless each time specifying iteration differently in the passed config. + config.set("save hofx", false); + config.set("save qc", false); + config.set("save obs errors", false); + config.set("iteration", std::to_string(iteration)); + for (size_t jj = 0; jj < nens; ++jj) { - hofx_.resetQc(); - hofx_.initialize(obsaux_, iteration); - // fill in geovals - std::vector getValuesConfig = - util::vectoriseAndFilter(obsconf_, "get values"); - - GetValuesPost_ getvals(obspaces_, hofx_.locations(), hofx_.requiredVars(), getValuesConfig); - getvals.fill(ens_xx[jj]); - // compute H(x) on filled in geovals and run the filters - obsens[jj] = hofx_.compute(getvals.geovals()); + computeHofX4D(config, ens_xx[jj], obsens[jj]); Log::test() << "H(x) for member " << jj+1 << ":" << std::endl << obsens[jj] << std::endl; obsens[jj].save("hofx"+std::to_string(iteration)+"_"+std::to_string(jj+1)); } - // QC flags and Obs errors are set to that of the last ensemble member - // TODO(someone) combine qc flags from all ensemble members - hofx_.saveQcFlags("EffectiveQC"); - hofx_.maskObsErrors(); - hofx_.saveObsErrors("ObsError"); + + // Compute H(mean(Xb)) + State4D_ xx_mean = ens_xx.mean(); + Observations_ y_mean_xb(obspaces_); + + // set QC for the mean + config.set("save qc", true); + config.set("save obs errors", true); + + computeHofX4D(config, xx_mean, y_mean_xb); + + y_mean_xb.save("hofx_y_mean_xb"+std::to_string(iteration)); + + // QC flags and Obs errors are set to that of the H(mean(Xb)) + R_->save("ObsError"); } - R_.reset(new ObsErrors_(obsconf_, obspaces_)); + // set inverse variances invVarR_.reset(new Departures_(R_->inverseVariance())); - invVarR_->mask(hofx_.qcflags()); - // calculate H(x) ensemble mean Observations_ yb_mean(obsens.mean()); // calculate H(x) ensemble perturbations for (size_t iens = 0; iens < nens; ++iens) { Yb_[iens] = obsens[iens] - yb_mean; - Yb_[iens].mask(hofx_.qcflags()); + Yb_[iens].mask(*invVarR_); } // calculate obs departures and mask with qc flag Observations_ yobs(obspaces_, "ObsValue"); omb_ = yobs - yb_mean; - omb_.mask(hofx_.qcflags()); + omb_.mask(*invVarR_); // return mean H(x) return yb_mean; @@ -185,16 +246,18 @@ template class LocalEnsembleSolverFactory { typedef Geometry Geometry_; typedef ObsSpaces ObsSpaces_; + typedef State4D State4D_; public: static std::unique_ptr> create(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, - size_t); + size_t, const State4D_ &); virtual ~LocalEnsembleSolverFactory() = default; protected: explicit LocalEnsembleSolverFactory(const std::string &); private: virtual LocalEnsembleSolver * make(ObsSpaces_ &, const Geometry_ &, - const eckit::Configuration &, size_t) = 0; + const eckit::Configuration &, size_t, + const State4D_ &) = 0; static std::map < std::string, LocalEnsembleSolverFactory * > & getMakers() { static std::map < std::string, LocalEnsembleSolverFactory * > makers_; return makers_; @@ -207,10 +270,12 @@ template class LocalEnsembleSolverMaker : public LocalEnsembleSolverFactory { typedef Geometry Geometry_; typedef ObsSpaces ObsSpaces_; + typedef State4D State4D_; virtual LocalEnsembleSolver * make(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & conf, size_t nens) - { return new T(obspaces, geometry, conf, nens); } + const eckit::Configuration & conf, size_t nens, + const State4D_ & xbmean) + { return new T(obspaces, geometry, conf, nens, xbmean); } public: explicit LocalEnsembleSolverMaker(const std::string & name) : LocalEnsembleSolverFactory(name) {} @@ -231,7 +296,8 @@ LocalEnsembleSolverFactory::LocalEnsembleSolverFactory(const std::st template std::unique_ptr> LocalEnsembleSolverFactory::create(ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & conf, size_t nens) { + const eckit::Configuration & conf, size_t nens, + const State4D_ & xbmean) { Log::trace() << "LocalEnsembleSolver::create starting" << std::endl; const std::string id = conf.getString("local ensemble DA.solver"); typename std::map*>::iterator @@ -246,7 +312,7 @@ LocalEnsembleSolverFactory::create(ObsSpaces_ & obspaces, const Geom throw std::runtime_error(id + " does not exist in local ensemble solver factory."); } std::unique_ptr> - ptr(jloc->second->make(obspaces, geometry, conf, nens)); + ptr(jloc->second->make(obspaces, geometry, conf, nens, xbmean)); Log::trace() << "LocalEnsembleSolver::create done" << std::endl; return ptr; } diff --git a/src/oops/assimilation/Minimizer.h b/src/oops/assimilation/Minimizer.h index b74d125a5..ddd2b58d0 100644 --- a/src/oops/assimilation/Minimizer.h +++ b/src/oops/assimilation/Minimizer.h @@ -23,8 +23,7 @@ #include "oops/assimilation/HMatrix.h" #include "oops/assimilation/HtMatrix.h" #include "oops/assimilation/MinimizerUtils.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" diff --git a/src/oops/assimilation/MinimizerUtils.h b/src/oops/assimilation/MinimizerUtils.h index 36634608a..a6b32c255 100644 --- a/src/oops/assimilation/MinimizerUtils.h +++ b/src/oops/assimilation/MinimizerUtils.h @@ -8,11 +8,14 @@ #ifndef OOPS_ASSIMILATION_MINIMIZERUTILS_H_ #define OOPS_ASSIMILATION_MINIMIZERUTILS_H_ +#include #include #include "eckit/config/Configuration.h" #include "oops/assimilation/ControlIncrement.h" +#include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/util/Logger.h" namespace oops { @@ -39,7 +42,7 @@ void writeIncrement(const eckit::Configuration & config, // print log Log::info() << "Write Increment - starting: " << loop << std::endl << std::endl; - const eckit::LocalConfiguration incConf(config, "increment"); + const eckit::LocalConfiguration incConf(onlineDiag, "increment"); // write increment dx.write(incConf); @@ -65,7 +68,7 @@ void writeKrylovBasis(const eckit::Configuration & config, // print log Log::info() << "Write Krylov Basis: starting: " << loop << std::endl; - eckit::LocalConfiguration basisConf(config, "krylov basis"); + eckit::LocalConfiguration basisConf(diagConf, "krylov basis"); basisConf.set("iteration", loop); // write increment @@ -76,6 +79,70 @@ void writeKrylovBasis(const eckit::Configuration & config, } } } + +//------------------------------------------------------------------------------------------------- +template +void writeEigenvectors(const eckit::Configuration & diagConf, + const std::vector & diag, + const std::vector & sub, + const std::vector & rhs, + std::vector>> & zvecs, + std::vector>> & hvecs, + const HtRinvHMatrix & HtRinvH, + ControlIncrement & temp, + ControlIncrement & eigenv, + ControlIncrement & eigenz) { + if (diagConf.has("online diagnostics")) { + int maxEigen = diagConf.getInt("online diagnostics.max eigenvectors", 0); + + const double nn = rhs.size(); + Eigen::MatrixXd TT = Eigen::MatrixXd::Zero(nn, nn); + + for (int ii = 0; ii < nn; ++ii) { + TT(ii, ii) = diag[ii]; + if (ii > 0) TT(ii-1, ii) = sub[ii-1]; + if (ii < nn - 1) TT(ii+1, ii) = sub[ii]; + } + + Eigen::SelfAdjointEigenSolver soluce(TT); + Eigen::MatrixXd eigenvecT = soluce.eigenvectors().real(); + Eigen::VectorXd eigenvalT = soluce.eigenvalues().real(); + + // Compute the eigenvectors (y = Zx) + for (int ii = 0; ii < maxEigen && ii < nn; ++ii) { + eigenz.zero(); + eigenv.zero(); + for (unsigned int jj = 0; jj < nn; ++jj) { + temp.zero(); + temp = *zvecs[jj]; + temp *= eigenvecT(jj, nn - 1 - ii); + eigenz += temp; + temp.zero(); + temp = *hvecs[jj]; + temp *= eigenvecT(jj, nn - 1 - ii); + eigenv += temp; + } + // Save the eigenvector + eckit::LocalConfiguration basisConf(diagConf, "online diagnostics.eigenvector"); + basisConf.set("iteration", ii); + eigenz.write(basisConf); + + // Verification that eigenz is an eigenvector: + // A.eigenv = eigenv + HtRinvH.Beigenv = eigenv + HtRinvH.eigenz = lambda eigenv + temp.zero(); + HtRinvH.multiply(eigenz, temp); + temp += eigenv; + eigenv *= eigenvalT(nn - 1 - ii); + temp -= eigenv; + Log::info() << "Eigenvalue " << ii+1 << " : " << eigenvalT(nn - 1 - ii) << std::endl; + Log::info() << "Norm A*y-lambda*y = " << dot_product(temp, temp) << std::endl; + + Log::test() << "Eigenvalue " << ii+1 << " : " << eigenvalT(nn - 1 - ii) << std::endl; + Log::test() << "Norm eigenvector = " << dot_product(eigenz, eigenz) << std::endl; + } // end for() + } // end if() +} + // ----------------------------------------------------------------------------- } // namespace oops diff --git a/src/oops/assimilation/PrimalMinimizer.h b/src/oops/assimilation/PrimalMinimizer.h index 1d7ceec17..d43ec3c22 100644 --- a/src/oops/assimilation/PrimalMinimizer.h +++ b/src/oops/assimilation/PrimalMinimizer.h @@ -11,6 +11,7 @@ #ifndef OOPS_ASSIMILATION_PRIMALMINIMIZER_H_ #define OOPS_ASSIMILATION_PRIMALMINIMIZER_H_ +#include #include #include @@ -74,6 +75,9 @@ PrimalMinimizer::doMinimize(const eckit::Configuration & config) { Hessian_ hessian(J_, runOnlineAdjTest); Bmat_ B(J_); +// Define minimisation starting point + CtrlInc_ * dx = new CtrlInc_(J_.jb()); + // Compute RHS CtrlInc_ rhs(J_.jb()); if (config.has("fsoi")) { @@ -87,8 +91,14 @@ PrimalMinimizer::doMinimize(const eckit::Configuration & config) { rhs *= -1.0; Log::info() << classname() << " rhs" << rhs << std::endl; -// Define minimisation starting point - CtrlInc_ * dx = new CtrlInc_(J_.jb()); +// Check for zero gradient (for example if no obs) + const double gnorm = dot_product(rhs, rhs); + const double epsilon = config.getDouble("epsilon", std::numeric_limits::epsilon()); + Log::info() << "Initial RHS squared norm = " << gnorm << std::endl; + if (gnorm < epsilon) { + Log::info() << "RHS smaller than " << epsilon << ", returning." << std::endl; + return dx; + } // Solve the linear system double reduc = this->solve(*dx, rhs, hessian, B, ninner, gnreduc); diff --git a/src/oops/assimilation/RinvHMatrix.h b/src/oops/assimilation/RinvHMatrix.h index 8c15918e2..2d6316c2c 100644 --- a/src/oops/assimilation/RinvHMatrix.h +++ b/src/oops/assimilation/RinvHMatrix.h @@ -21,7 +21,6 @@ #include "oops/assimilation/DualVector.h" #include "oops/base/Departures.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Increment.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" #include "oops/util/PrintAdjTest.h" diff --git a/src/oops/assimilation/SaddlePointPrecondMatrix.h b/src/oops/assimilation/SaddlePointPrecondMatrix.h index ffc2e8a20..71d85ba35 100644 --- a/src/oops/assimilation/SaddlePointPrecondMatrix.h +++ b/src/oops/assimilation/SaddlePointPrecondMatrix.h @@ -18,7 +18,6 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DualVector.h" #include "oops/assimilation/SaddlePointVector.h" -#include "oops/interface/Increment.h" namespace oops { diff --git a/src/oops/base/Accumulator.h b/src/oops/base/Accumulator.h index a09e52308..2e1411ad0 100644 --- a/src/oops/base/Accumulator.h +++ b/src/oops/base/Accumulator.h @@ -11,8 +11,8 @@ #ifndef OOPS_BASE_ACCUMULATOR_H_ #define OOPS_BASE_ACCUMULATOR_H_ +#include "oops/base/Geometry.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" namespace oops { diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index 665b59508..aea5532b8 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -21,8 +21,8 @@ #include "oops/base/GeneralizedDepartures.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/ObsVector.h" #include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsVector.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" @@ -74,11 +74,13 @@ class Departures : public GeneralizedDepartures { /// Mask out departures where the passed in qc flags are > 0 void mask(ObsDataVec_); +/// Mask out departures where \p mask has missing values + void mask(const Departures & mask); /// Pack departures in an Eigen vector (excluding departures that are masked out) - Eigen::VectorXd packEigen(const ObsDataVec_ &) const; + Eigen::VectorXd packEigen(const Departures &) const; /// Size of departures packed into an Eigen vector - size_t packEigenSize(const ObsDataVec_ &) const; + size_t packEigenSize(const Departures &) const; /// Save departures values void save(const std::string &) const; @@ -210,28 +212,35 @@ void Departures::mask(ObsDataVec_ qcflags) { } } // ----------------------------------------------------------------------------- +template +void Departures::mask(const Departures & mask) { + for (size_t ii = 0; ii < dep_.size(); ++ii) { + dep_[ii].mask(mask[ii]); + } +} +// ----------------------------------------------------------------------------- template -Eigen::VectorXd Departures::packEigen(const ObsDataVec_ & mask) const { +Eigen::VectorXd Departures::packEigen(const Departures & mask) const { std::vector len(dep_.size()); for (size_t idep = 0; idep < dep_.size(); ++idep) { - len[idep] = dep_[idep].packEigenSize(*mask[idep]); + len[idep] = dep_[idep].packEigenSize(mask[idep]); } size_t all_len = std::accumulate(len.begin(), len.end(), 0); Eigen::VectorXd vec(all_len); size_t ii = 0; for (size_t idep = 0; idep < dep_.size(); ++idep) { - vec.segment(ii, len[idep]) = dep_[idep].packEigen(*mask[idep]); + vec.segment(ii, len[idep]) = dep_[idep].packEigen(mask[idep]); ii += len[idep]; } return vec; } // ----------------------------------------------------------------------------- template -size_t Departures::packEigenSize(const ObsDataVec_ & mask) const { +size_t Departures::packEigenSize(const Departures & mask) const { size_t len = 0; for (size_t idep = 0; idep < dep_.size(); ++idep) { - len += dep_[idep].packEigenSize(*mask[idep]); + len += dep_[idep].packEigenSize(mask[idep]); } return len; } diff --git a/src/oops/base/DeparturesEnsemble.h b/src/oops/base/DeparturesEnsemble.h index 12fe0a120..2e069339c 100644 --- a/src/oops/base/DeparturesEnsemble.h +++ b/src/oops/base/DeparturesEnsemble.h @@ -38,7 +38,7 @@ template class DeparturesEnsemble { const Departures_ & operator[](const size_t ii) const {return ensemblePerturbs_[ii];} /// pack ensemble of dep. as contiguous block of memory - Eigen::MatrixXd packEigen(const ObsDataVec_ &) const; + Eigen::MatrixXd packEigen(const Departures_ &) const; private: std::vector ensemblePerturbs_; // ensemble perturbations @@ -59,7 +59,7 @@ DeparturesEnsemble::DeparturesEnsemble(const ObsSpaces_ & obsdb, const size // ----------------------------------------------------------------------------- template -Eigen::MatrixXd DeparturesEnsemble::packEigen(const ObsDataVec_ & mask) const { +Eigen::MatrixXd DeparturesEnsemble::packEigen(const Departures_ & mask) const { std::size_t myNobs = ensemblePerturbs_[0].packEigenSize(mask); std::size_t myNens = ensemblePerturbs_.size(); diff --git a/src/oops/base/EnsembleCovariance.h b/src/oops/base/EnsembleCovariance.h index 7005951c7..1d6d3e40d 100644 --- a/src/oops/base/EnsembleCovariance.h +++ b/src/oops/base/EnsembleCovariance.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -20,14 +20,14 @@ #include "eckit/system/ResourceUsage.h" #include "oops/assimilation/GMRESR.h" +#include "oops/base/Geometry.h" #include "oops/base/IdentityMatrix.h" +#include "oops/base/Increment.h" #include "oops/base/IncrementEnsemble.h" -#include "oops/base/LocalizationBase.h" +#include "oops/base/Localization.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" @@ -41,7 +41,7 @@ class EnsembleCovariance : public ModelSpaceCovarianceBase, private util::ObjectCounter> { typedef Geometry Geometry_; typedef Increment Increment_; - typedef LocalizationBase Localization_; + typedef Localization Localization_; typedef State State_; typedef IncrementEnsemble Ensemble_; typedef std::shared_ptr> EnsemblePtr_; @@ -78,7 +78,7 @@ EnsembleCovariance::EnsembleCovariance(const Geometry_ & resol, const Var ens_.reset(new Ensemble_(conf, xb, fg, resol, vars)); if (conf.has("localization")) { const eckit::LocalConfiguration confloc(conf, "localization"); - loc_ = LocalizationFactory::create(resol, xb.validTime(), confloc); + loc_.reset(new Localization_(resol, confloc)); } size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); this->setObjectSize(current - init); diff --git a/src/oops/base/Geometry.h b/src/oops/base/Geometry.h new file mode 100644 index 000000000..4c1239d30 --- /dev/null +++ b/src/oops/base/Geometry.h @@ -0,0 +1,83 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2021- UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_BASE_GEOMETRY_H_ +#define OOPS_BASE_GEOMETRY_H_ + +#include +#include +#include + +#include "oops/interface/Geometry.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief Geometry class used in oops; subclass of interface class interface::Geometry. +/// +/// \details Handles additional MPI communicator parameter in the constructors +/// (for MPI distribution in time, used in oops for 4DEnVar and weak-constraint 4DVar). +/// Adds extra methods that do not need to be implemented in the implementations: +/// - timeComm() (accessor to the MPI communicator in time) +template +class Geometry : public interface::Geometry { + typedef typename MODEL::Geometry Geometry_; + public: + typedef typename interface::Geometry::Parameters_ Parameters_; + + /// Constructor from Parameters and mpi communicators: \p geometry for spatial distribution + /// (handled by the implementation) and \p time for distribution in time (handled by oops) + Geometry(const Parameters_ &, const eckit::mpi::Comm & geometry, + const eckit::mpi::Comm & time); + /// Constructor from Configuration and mpi communicators: \p geometry for spatial distribution + /// (handled by the implementation) and \p time for distribution in time (handled by oops) + Geometry(const eckit::Configuration &, const eckit::mpi::Comm & geometry, + const eckit::mpi::Comm & time = oops::mpi::myself()); + /// Constructor from pointer to the MODEL::Geometry (used in 1DVar filter) + explicit Geometry(std::shared_ptr); + + /// Accessor to the MPI communicator for distribution in time + const eckit::mpi::Comm & timeComm() const {return *timeComm_;} + + private: + const eckit::mpi::Comm * timeComm_; /// pointer to the MPI communicator in time +}; + +// ----------------------------------------------------------------------------- + +template +Geometry::Geometry(const eckit::Configuration & config, + const eckit::mpi::Comm & geometry, const eckit::mpi::Comm & time): + interface::Geometry(config, geometry), timeComm_(&time) +{} + +// ----------------------------------------------------------------------------- + +template +Geometry::Geometry(const Parameters_ & parameters, + const eckit::mpi::Comm & geometry, const eckit::mpi::Comm & time): + interface::Geometry(parameters, geometry), timeComm_(&time) +{} + +// ----------------------------------------------------------------------------- + +template +Geometry::Geometry(std::shared_ptr ptr): + interface::Geometry(ptr), timeComm_(&oops::mpi::myself()) +{} + +} // namespace oops + +#endif // OOPS_BASE_GEOMETRY_H_ diff --git a/src/oops/base/GetValuePost.h b/src/oops/base/GetValuePost.h index bdf38bd0c..fc9131764 100644 --- a/src/oops/base/GetValuePost.h +++ b/src/oops/base/GetValuePost.h @@ -14,12 +14,12 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -65,6 +65,8 @@ class GetValuePost { const Variables geovars_; /// Variables needed from model GetValues_ getvals_; /// GetValues used to fill in GeoVaLs std::unique_ptr geovals_; /// GeoVaLs that are filled in + std::vector sizes_; /// Sizes (e.g. number of vertical levels) + /// for all Variables in GeoVaLs bool initialized_; }; @@ -75,7 +77,8 @@ GetValuePost::GetValuePost(const eckit::Configuration & conf, const const util::DateTime & bgn, const util::DateTime & end, const Locations_ & locations, const Variables & vars) : winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), - getvals_(geom, locations_, conf), geovals_(), initialized_(false) + getvals_(geom, locations_, conf), geovals_(), sizes_(geom.variableSizes(geovars_)), + initialized_(false) { Log::trace() << "GetValuePost::GetValuePost" << std::endl; } @@ -86,7 +89,7 @@ template void GetValuePost::initialize(const util::Duration & tstep) { Log::trace() << "GetValuePost::doInitialize start" << std::endl; hslot_ = tstep/2; - geovals_.reset(new GeoVaLs_(locations_, geovars_)); + geovals_.reset(new GeoVaLs_(locations_, geovars_, sizes_)); initialized_ = true; Log::trace() << "GetValuePost::doInitialize done" << std::endl; } diff --git a/src/oops/base/GetValuePosts.h b/src/oops/base/GetValuePosts.h index d36ef7c30..1f11505ea 100644 --- a/src/oops/base/GetValuePosts.h +++ b/src/oops/base/GetValuePosts.h @@ -16,8 +16,8 @@ #include "oops/base/GetValuePost.h" #include "oops/base/PostBase.h" +#include "oops/base/State.h" #include "oops/interface/ChangeVariables.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -45,12 +45,13 @@ class GetValuePosts : public PostBase> { // Data std::vector getvals_; + Variables geovars_; }; // ----------------------------------------------------------------------------- template -GetValuePosts::GetValuePosts() : PostBase(), getvals_() { +GetValuePosts::GetValuePosts() : PostBase(), getvals_(), geovars_() { Log::trace() << "GetValuePosts::GetValuePosts" << std::endl; } @@ -60,6 +61,7 @@ template void GetValuePosts::append(GetValuePtr_ getval) { Log::trace() << "GetValuePosts::append start" << std::endl; getvals_.push_back(getval); + geovars_ += getval->requiredVariables(); Log::trace() << "GetValuePosts::append done" << std::endl; } @@ -78,8 +80,13 @@ void GetValuePosts::doInitialize(const State_ &, const util::DateTim template void GetValuePosts::doProcessing(const State_ & xx) { Log::trace() << "GetValuePosts::doProcessing start" << std::endl; -// Change of variables will go here - for (GetValuePtr_ getval : getvals_) getval->process(xx); + + eckit::LocalConfiguration chvarconf; // empty for now + ChangeVariables_ chvar(chvarconf, xx.geometry(), xx.variables(), geovars_); + State_ zz(xx.geometry(), geovars_, xx.validTime()); + chvar.changeVar(xx, zz); + + for (GetValuePtr_ getval : getvals_) getval->process(zz); Log::trace() << "GetValuePosts::doProcessing done" << std::endl; } diff --git a/src/oops/base/GetValueTLAD.h b/src/oops/base/GetValueTLAD.h index fcc455965..dc2002309 100644 --- a/src/oops/base/GetValueTLAD.h +++ b/src/oops/base/GetValueTLAD.h @@ -14,14 +14,14 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" -#include "oops/interface/Increment.h" #include "oops/interface/LinearGetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -77,11 +77,15 @@ class GetValueTLAD { util::DateTime winend_; /// End of assimilation window util::Duration hslot_; /// Half time slot - const Locations_ & locations_; /// locations of observations - const Variables geovars_; /// Variables needed from model - const Variables linvars_; /// Variables needed from linear model - GetValues_ getvals_; /// GetValues used to fill in GeoVaLs - std::unique_ptr geovals_; /// GeoVaLs that are filled in + const Locations_ & locations_; /// locations of observations + const Variables geovars_; /// Variables needed from model + const std::vector geovars_sizes_; /// Sizes (e.g. number of vertical levels) + /// for all geovars_ variables + const Variables linvars_; /// Variables needed from linear model + const std::vector linvars_sizes_; /// Sizes (e.g. number of vertical levels) + /// for all linvars_ variables + GetValues_ getvals_; /// GetValues used to fill in GeoVaLs + std::unique_ptr geovals_; /// GeoVaLs that are filled in std::unique_ptr gvalsad_; /// Input GeoVaLs for adjoint forcing }; @@ -92,7 +96,9 @@ GetValueTLAD::GetValueTLAD(const eckit::Configuration & conf, const const util::DateTime & bgn, const util::DateTime & end, const Locations_ & locations, const Variables & vars, const Variables & varl) - : winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), linvars_(varl), + : winbgn_(bgn), winend_(end), hslot_(), locations_(locations), + geovars_(vars), geovars_sizes_(geom.variableSizes(geovars_)), + linvars_(varl), linvars_sizes_(geom.variableSizes(linvars_)), getvals_(geom, locations_, conf), geovals_(), gvalsad_(nullptr) { Log::trace() << "GetValueTLAD::GetValueTLAD" << std::endl; @@ -104,7 +110,7 @@ template void GetValueTLAD::initializeTraj(const util::Duration & tstep) { Log::trace() << "GetValueTLAD::initializeTraj start" << std::endl; hslot_ = tstep/2; - geovals_.reset(new GeoVaLs_(locations_, geovars_)); + geovals_.reset(new GeoVaLs_(locations_, geovars_, geovars_sizes_)); Log::trace() << "GetValueTLAD::initializeTraj done" << std::endl; } @@ -128,7 +134,7 @@ template void GetValueTLAD::initializeTL(const util::Duration & tstep) { Log::trace() << "GetValueTLAD::initializeTL start" << std::endl; hslot_ = tstep/2; - geovals_.reset(new GeoVaLs_(locations_, linvars_)); + geovals_.reset(new GeoVaLs_(locations_, linvars_, linvars_sizes_)); Log::trace() << "GetValueTLAD::initializeTL done" << std::endl; } diff --git a/src/oops/base/GetValueTLADs.h b/src/oops/base/GetValueTLADs.h index 53c281f32..91bba0158 100644 --- a/src/oops/base/GetValueTLADs.h +++ b/src/oops/base/GetValueTLADs.h @@ -12,9 +12,9 @@ #include #include "oops/base/GetValueTLAD.h" +#include "oops/base/Increment.h" #include "oops/base/PostBaseTLAD.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/GetValuesPost.h b/src/oops/base/GetValuesPost.h index 9631b1571..1bb59dcbe 100644 --- a/src/oops/base/GetValuesPost.h +++ b/src/oops/base/GetValuesPost.h @@ -17,14 +17,15 @@ #include #include -#include "oops/assimilation/State4D.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostBase.h" +#include "oops/base/State.h" +#include "oops/base/State4D.h" #include "oops/base/Variables.h" +#include "oops/interface/ChangeVariables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -40,6 +41,7 @@ namespace oops { /// - as a method fill() on State4D template class GetValuesPost : public PostBase> { + typedef ChangeVariables ChangeVariables_; typedef GeoVaLs GeoVaLs_; typedef Locations Locations_; typedef ObsSpaces ObsSpaces_; @@ -127,7 +129,8 @@ void GetValuesPost::doInitialize(const State_ & xx, const util::Date for (size_t jj = 0; jj < locations_.size(); ++jj) { getvals_.emplace_back(new GetValues_(xx.geometry(), *locations_[jj], getvalsconfs_[jj])); - geovals_.emplace_back(new GeoVaLs_(*locations_[jj], geovars_[jj])); + geovals_.emplace_back(new GeoVaLs_(*locations_[jj], geovars_[jj], + xx.geometry().variableSizes(geovars_[jj]))); } Log::trace() << "GetValuesPost::doInitialize done" << std::endl; @@ -141,9 +144,16 @@ void GetValuesPost::doProcessing(const State_ & xx) { util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); util::DateTime t2 = std::min(xx.validTime()+hslot_, winend_); + eckit::LocalConfiguration chvarconf; // empty for now + Variables gvars; + for (size_t jj = 0; jj < getvals_.size(); ++jj) gvars += geovars_[jj]; + ChangeVariables_ chvar(chvarconf, xx.geometry(), xx.variables(), gvars); + State_ zz(xx.geometry(), gvars, xx.validTime()); + chvar.changeVar(xx, zz); + // Get state variables at obs locations for (size_t jj = 0; jj < getvals_.size(); ++jj) { - getvals_[jj]->fillGeoVaLs(xx, t1, t2, *geovals_[jj]); + getvals_[jj]->fillGeoVaLs(zz, t1, t2, *geovals_[jj]); } Log::trace() << "GetValuesPost::doProcessing done" << std::endl; } diff --git a/src/oops/base/HybridCovariance.h b/src/oops/base/HybridCovariance.h index 27460cd9d..8d8987335 100644 --- a/src/oops/base/HybridCovariance.h +++ b/src/oops/base/HybridCovariance.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -19,13 +19,12 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "oops/assimilation/GMRESR.h" -#include "oops/base/EnsembleCovariance.h" +#include "oops/base/Geometry.h" #include "oops/base/IdentityMatrix.h" +#include "oops/base/Increment.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" diff --git a/src/oops/base/Increment.h b/src/oops/base/Increment.h new file mode 100644 index 000000000..6a85f9bd9 --- /dev/null +++ b/src/oops/base/Increment.h @@ -0,0 +1,217 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2017-2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_BASE_INCREMENT_H_ +#define OOPS_BASE_INCREMENT_H_ + +#include "atlas/field.h" + +#include "oops/base/Geometry.h" +#include "oops/base/State.h" +#include "oops/base/Variables.h" +#include "oops/interface/Increment.h" +#include "oops/mpi/mpi.h" +#include "oops/util/DateTime.h" +#include "oops/util/gatherPrint.h" +#include "oops/util/Timer.h" + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief Increment class used in oops +/// +/// \details +/// Adds extra methods that do not need to be implemented in the model implementations: +/// - timeComm() (accessor to the MPI communicator in time - collection of processes +/// holding the data needed to represent the state in a particular region +/// of space X_i and throughout the whole time interval for which DA is done) +/// - variables() (accessor to variables in this Increment) +/// - shift_forward +/// - shift_backward +/// - toAtlas, atlas +/// +/// Adds communication through time to the following Increment methods: +/// - dot_product_with +/// - norm +/// - print + +template +class Increment : public interface::Increment { + typedef Geometry Geometry_; + + public: + /// Constructor for specified \p geometry, with \p variables, valid on \p date + Increment(const Geometry_ & geometry, const Variables & variables, const util::DateTime & date); + /// Copies \p other increment, changing its resolution to \p geometry + Increment(const Geometry_ & geometry, const Increment & other); + /// Creates Increment with the same geometry and variables as \p other. + /// Copies \p other if \p copy is true, otherwise creates zero increment + Increment(const Increment & other, const bool copy = true); + + /// Accessor to the time communicator + const eckit::mpi::Comm & timeComm() const {return *timeComm_;} + /// Accessor to Variables stored in this increment + const Variables & variables() const {return variables_;} + + /// Shift forward in time by \p dt + void shift_forward(const util::DateTime & dt); + /// Shift backward in time by \p dt + void shift_backward(const util::DateTime & dt); + + /// Set ATLAS fieldset associated with this Increment internally + void toAtlas(); + /// Allow to access base class's method as well + using interface::Increment::toAtlas; + /// Accessors to the ATLAS fieldset + atlas::FieldSet & atlas() {return atlasFieldSet_;} + const atlas::FieldSet & atlas() const {return atlasFieldSet_;} + + /// dot product with the \p other increment + double dot_product_with(const Increment & other) const; + /// Norm for diagnostics + double norm() const; + + private: + void print(std::ostream &) const override; + + Variables variables_; /// Variables stored in this Increment + const eckit::mpi::Comm * timeComm_; /// pointer to the MPI communicator in time + atlas::FieldSet atlasFieldSet_; /// Atlas fields associated with this Increment +}; + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Geometry_ & geometry, const Variables & variables, + const util::DateTime & date): + interface::Increment(geometry, variables, date), variables_(variables), + timeComm_(&geometry.timeComm()) +{} + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Geometry_ & geometry, const Increment & other): + interface::Increment(geometry, other), variables_(other.variables_), + timeComm_(other.timeComm_) +{} + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Increment & other, const bool copy): + interface::Increment(other, copy), variables_(other.variables_), + timeComm_(other.timeComm_) +{} + +// ----------------------------------------------------------------------------- + +template +double Increment::dot_product_with(const Increment & dx) const { + double zz = interface::Increment::dot_product_with(dx); + timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + return zz; +} + +// ----------------------------------------------------------------------------- + +template +double Increment::norm() const { + double zz = interface::Increment::norm(); + zz *= zz; + timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + zz = sqrt(zz); + return zz; +} + +// ----------------------------------------------------------------------------- + +template +void Increment::shift_forward(const util::DateTime & begin) { + Log::trace() << "Increment::Increment shift_forward starting" << std::endl; + static int tag = 159357; + size_t mytime = timeComm_->rank(); + +// Send values of M.dx_i at end of my subwindow to next subwindow + if (mytime + 1 < timeComm_->size()) { + oops::mpi::send(*timeComm_, *this, mytime+1, tag); + } + +// Receive values at beginning of my subwindow from previous subwindow + if (mytime > 0) { + oops::mpi::receive(*timeComm_, *this, mytime-1, tag); + } else { + this->zero(begin); + } + + ++tag; + Log::trace() << "Increment::Increment shift_forward done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void Increment::shift_backward(const util::DateTime & end) { + Log::trace() << "Increment::Increment shift_backward starting" << std::endl; + static int tag = 30951; + size_t mytime = timeComm_->rank(); + +// Send values of dx_i at start of my subwindow to previous subwindow + if (mytime > 0) { + oops::mpi::send(*timeComm_, *this, mytime-1, tag); + } + +// Receive values at end of my subwindow from next subwindow + if (mytime + 1 < timeComm_->size()) { + oops::mpi::receive(*timeComm_, *this, mytime+1, tag); + } else { + this->zero(end); + } + + ++tag; + Log::trace() << "Increment::Increment shift_backward done" << std::endl; +} + + +// ----------------------------------------------------------------------------- +template +void Increment::toAtlas() { + interface::Increment::setAtlas(&atlasFieldSet_); + interface::Increment::toAtlas(&atlasFieldSet_); + this->increment_.reset(); +} + +// ----------------------------------------------------------------------------- + +template +void Increment::print(std::ostream & os) const { + if (timeComm_->size() > 1) { + gatherPrint(os, this->increment(), *timeComm_); + } else { + os << this->increment(); + } +} + +// ----------------------------------------------------------------------------- +/// Add on \p dx incrment to model state \p xx +template +State & operator+=(State & xx, const Increment & dx) { + Log::trace() << "operator+=(State, Increment) starting" << std::endl; + util::Timer timer("oops::Increment", "operator+=(State, Increment)"); + xx.state() += dx.increment(); + Log::trace() << "operator+=(State, Increment) done" << std::endl; + return xx; +} + + +} // namespace oops + +#endif // OOPS_BASE_INCREMENT_H_ diff --git a/src/oops/assimilation/Increment4D.h b/src/oops/base/Increment4D.h similarity index 95% rename from src/oops/assimilation/Increment4D.h rename to src/oops/base/Increment4D.h index 00c5b686e..2820052fd 100644 --- a/src/oops/assimilation/Increment4D.h +++ b/src/oops/base/Increment4D.h @@ -8,8 +8,8 @@ * does it submit to any jurisdiction. */ -#ifndef OOPS_ASSIMILATION_INCREMENT4D_H_ -#define OOPS_ASSIMILATION_INCREMENT4D_H_ +#ifndef OOPS_BASE_INCREMENT4D_H_ +#define OOPS_BASE_INCREMENT4D_H_ #include #include @@ -17,10 +17,10 @@ #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/State4D.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State4D.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" @@ -140,4 +140,4 @@ void Increment4D::schur_product_with(const Increment4D & x2) { // ----------------------------------------------------------------------------- } // namespace oops -#endif // OOPS_ASSIMILATION_INCREMENT4D_H_ +#endif // OOPS_BASE_INCREMENT4D_H_ diff --git a/src/oops/base/IncrementEnsemble.h b/src/oops/base/IncrementEnsemble.h index 6add14a10..d05e3b31e 100644 --- a/src/oops/base/IncrementEnsemble.h +++ b/src/oops/base/IncrementEnsemble.h @@ -20,12 +20,12 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/base/Accumulator.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/State.h" #include "oops/base/StateEnsemble.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" @@ -48,9 +48,6 @@ template class IncrementEnsemble { /// Constructor IncrementEnsemble(const Geometry_ & resol, const Variables & vars, const util::DateTime &, const int rank); - /// \brief construct ensemble of perturbations as \p ens - \p mean; holding - // \p vars variables - IncrementEnsemble(const StateEnsemble_ & ens, const State_ & mean, const Variables & vars); IncrementEnsemble(const eckit::Configuration &, const State_ &, const State_ &, const Geometry_ &, const Variables &); /// \brief construct ensemble of perturbations by reading them from disk @@ -60,6 +57,8 @@ template class IncrementEnsemble { IncrementEnsemble(const Geometry_ &, const Variables &, const eckit::Configuration &, const eckit::Configuration &); + void write(const eckit::Configuration &) const; + /// Accessors size_t size() const {return ensemblePerturbs_.size();} Increment_ & operator[](const int ii) {return ensemblePerturbs_[ii];} @@ -68,10 +67,6 @@ template class IncrementEnsemble { /// Control variables const Variables & controlVariables() const {return vars_;} - /// Release / reset - void releaseMember(); - void appendMember(const Increment_ &); - private: const Variables vars_; std::vector ensemblePerturbs_; @@ -91,21 +86,6 @@ IncrementEnsemble::IncrementEnsemble(const Geometry_ & resol, const Varia Log::trace() << "IncrementEnsemble:contructor done" << std::endl; } -// ==================================================================================== - -template -IncrementEnsemble::IncrementEnsemble(const StateEnsemble_ & ensemble, - const State_ & mean, const Variables & vars) - : vars_(vars), ensemblePerturbs_() -{ - ensemblePerturbs_.reserve(ensemble.size()); - for (size_t ii = 0; ii < ensemble.size(); ++ii) { - ensemblePerturbs_.emplace_back(ensemble[ii].geometry(), vars, ensemble[ii].validTime()); - ensemblePerturbs_[ii].diff(ensemble[ii], mean); - } - Log::trace() << "IncrementEnsemble:contructor(StateEnsemble) done" << std::endl; -} - // ----------------------------------------------------------------------------- template @@ -223,15 +203,13 @@ IncrementEnsemble::IncrementEnsemble(const Geometry_ & resol, const Varia // ----------------------------------------------------------------------------- template -void IncrementEnsemble::releaseMember() { - ensemblePerturbs_.erase(ensemblePerturbs_.begin()); -} - -// ----------------------------------------------------------------------------- - -template -void IncrementEnsemble::appendMember(const Increment_ & dx) { - ensemblePerturbs_.emplace_back(dx); +void IncrementEnsemble::write(const eckit::Configuration & config) const +{ + eckit::LocalConfiguration outConfig(config); + for (size_t ii=0; ii < size(); ++ii) { + outConfig.set("member", ii+1); + ensemblePerturbs_[ii].write(outConfig); + } } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/IncrementEnsemble4D.h b/src/oops/base/IncrementEnsemble4D.h index ebd8f2b20..a95a9d3b7 100644 --- a/src/oops/base/IncrementEnsemble4D.h +++ b/src/oops/base/IncrementEnsemble4D.h @@ -16,11 +16,11 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/Increment4D.h" -#include "oops/assimilation/State4D.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment4D.h" +#include "oops/base/State4D.h" #include "oops/base/StateEnsemble4D.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" diff --git a/src/oops/interface/LinearModel.h b/src/oops/base/LinearModel.h similarity index 70% rename from src/oops/interface/LinearModel.h rename to src/oops/base/LinearModel.h index a089ac464..295dce45f 100644 --- a/src/oops/interface/LinearModel.h +++ b/src/oops/base/LinearModel.h @@ -1,32 +1,28 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * (C) Copyright 2018-2021 UCAR. + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_INTERFACE_LINEARMODEL_H_ -#define OOPS_INTERFACE_LINEARMODEL_H_ +#ifndef OOPS_BASE_LINEARMODEL_H_ +#define OOPS_BASE_LINEARMODEL_H_ #include #include +#include #include -#include "oops/base/LinearModelBase.h" -#include "oops/base/PostProcessor.h" -#include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" +#include "oops/generic/LinearModelBase.h" #include "oops/interface/Increment.h" #include "oops/interface/ModelAuxControl.h" #include "oops/interface/ModelAuxIncrement.h" -#include "oops/interface/State.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" -#include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -36,27 +32,13 @@ namespace eckit { namespace oops { -/// Encapsulates the linear forecast model. -/*! - * This class provides the operations associated with the LinearModel. It wraps - * the actual linear model which can be a model specific one or a generic one - * (identity). The interface for the linear model comprises two levels (LinearModel - * and LinearModelBase) because we want run time polymorphism. - * - * Note: implementations of this interface can opt to extract their settings either from - * a Configuration object or from a subclass of LinearModelParametersBase. - * - * In the former case, they should provide a constructor with the following signature: - * - * LinearModel(const Geometry_ &, const eckit::Configuration &); - * - * In the latter case, the implementer should first define a subclass of LinearModelParametersBase - * holding the settings of the model in question. The implementation of the LinearModel interface - * should then typedef `Parameters_` to the name of that subclass and provide a constructor with - * the following signature: - * - * LinearModel(const Geometry_ &, const Parameters_ &); - */ +/// \brief Abstract linear forecast model used by high level algorithms and applications. +/// +/// Note: to see methods that need to be implemented in a generic linear forecast model +/// implementation, see LinearModelBase class in generic/LinearModelBase.h. To see methods that need +/// to be implemented in a MODEL-specific linear forecast model implementation, see +/// interface::LinearModelBase class in interface/LinearModelBase.h. +/// // ----------------------------------------------------------------------------- template @@ -64,10 +46,10 @@ class LinearModel : public util::Printable, private boost::noncopyable, private util::ObjectCounter > { typedef LinearModelBase LinearModelBase_; - typedef Increment Increment_; typedef Geometry Geometry_; - typedef ModelAuxControl ModelAux_; - typedef ModelAuxIncrement ModelAuxIncr_; + typedef Increment Increment_; + typedef ModelAuxControl ModelAuxCtl_; + typedef ModelAuxIncrement ModelAuxInc_; typedef State State_; public: @@ -75,55 +57,65 @@ class LinearModel : public util::Printable, LinearModel(const Geometry_ &, const LinearModelParametersBase &); LinearModel(const Geometry_ &, const eckit::Configuration &); - ~LinearModel(); + virtual ~LinearModel(); -/// Run the tangent linear forecast - void forecastTL(Increment_ &, const ModelAuxIncr_ &, const util::Duration &, + /// \brief Run the linear forecast from increment \p dx for \p len time, with \p post + /// postprocessors. Does not need to be implemented in the subclasses of LinearModelBase + void forecastTL(Increment_ & dx, const ModelAuxInc_ &, + const util::Duration & len, PostProcessor post = PostProcessor(), PostProcessorTLAD cost = PostProcessorTLAD(), const bool idmodel = false) const; -/// Run the adjoint forecast - void forecastAD(Increment_ &, ModelAuxIncr_ &, const util::Duration &, + /// \brief Run the adjoint linear forecast from increment \p dx for \p len -time, with \p post + /// postprocessors. Note that the clock ticks backwards. Does not need to be implemented in the + /// subclasses of LinearModelBase + void forecastAD(Increment_ & dx, ModelAuxInc_ &, + const util::Duration & len, PostProcessor post = PostProcessor(), PostProcessorTLAD cost = PostProcessorTLAD(), const bool idmodel = false) const; -// Set the linearization trajectory - void setTrajectory(const State_ &, State_ &, const ModelAux_ &); + /// \brief Set the trajectory for the linear model + void setTrajectory(const State_ &, State_ &, const ModelAuxCtl_ &); -// Information and diagnostics - const util::Duration & timeResolution() const {return tlm_->timeResolution();} - const oops::Variables & variables() const {return tlm_->variables();} + /// \brief Time step for running LinearModel's forecast in oops (frequency with which the + /// State will be updated) + const util::Duration & timeResolution() const {return linearmodel_->timeResolution();} + /// \brief LinearModel variables (only used in 4DVar) + const oops::Variables & variables() const {return linearmodel_->variables();} - protected: -// Run the TL forecast + private: + /// \brief Tangent linear forecast initialization, called before every run void initializeTL(Increment_ &) const; - void stepTL(Increment_ &, const ModelAuxIncr_ &) const; + /// \brief Tangent linear forecast "step", called during run; updates increment to the next time + void stepTL(Increment_ &, const ModelAuxInc_ &) const; + /// \brief Tangent linear forecast finalization; called after each run void finalizeTL(Increment_ &) const; - -// Run the AD forecast + /// \brief Adjoint forecast initialization, called before every run void initializeAD(Increment_ &) const; - void stepAD(Increment_ &, ModelAuxIncr_ &) const; + /// \brief Adjoint forecast "step", called during run; updates increment to the next time + void stepAD(Increment_ &, ModelAuxInc_ &) const; + /// \brief Adjoint forecast finalization; called after each run void finalizeAD(Increment_ &) const; - private: -// diagnostics + /// \brief Print, used in logging void print(std::ostream &) const; - std::unique_ptr tlm_; + /// \brief Pointer to the LinearModel implementation + std::unique_ptr linearmodel_; }; // ============================================================================= template LinearModel::LinearModel(const Geometry_ & resol, const LinearModelParametersBase & params) - : tlm_() + : linearmodel_() { Log::trace() << "LinearModel::LinearModel starting" << std::endl; util::Timer timer(classname(), "LinearModel"); Log::info() << "LinearModel configuration is:" << params << std::endl; - tlm_.reset(LinearModelFactory::create(resol, params)); + linearmodel_.reset(LinearModelFactory::create(resol, params)); Log::trace() << "LinearModel::LinearModel done" << std::endl; } @@ -132,7 +124,7 @@ LinearModel::LinearModel(const Geometry_ & resol, const LinearModelParame template LinearModel::LinearModel(const Geometry_ & resol, const eckit::Configuration & conf) : LinearModel(resol, - validateAndDeserialize>(conf).modelParameters) + validateAndDeserialize>(conf).linearModelParameters) {} // ----------------------------------------------------------------------------- @@ -141,18 +133,14 @@ template LinearModel::~LinearModel() { Log::trace() << "LinearModel::~LinearModel starting" << std::endl; util::Timer timer(classname(), "~LinearModel"); - tlm_.reset(); + linearmodel_.reset(); Log::trace() << "LinearModel::~LinearModel done" << std::endl; } // ----------------------------------------------------------------------------- -// ----------------------------------------------------------------------------- -/// Run forecast TL and AD -// ----------------------------------------------------------------------------- - template -void LinearModel::forecastTL(Increment_ & dx, const ModelAuxIncr_ & mctl, +void LinearModel::forecastTL(Increment_ & dx, const ModelAuxInc_ & maux, const util::Duration & len, PostProcessor post, PostProcessorTLAD cost, @@ -160,7 +148,7 @@ void LinearModel::forecastTL(Increment_ & dx, const ModelAuxIncr_ & mctl, Log::trace() << "LinearModel::forecastTL starting" << std::endl; const util::DateTime end(dx.validTime() + len); - const util::Duration tstep(tlm_->timeResolution()); + const util::Duration tstep(linearmodel_->timeResolution()); Log::info() << "LinearModel::forecastTL: Starting " << dx << std::endl; this->initializeTL(dx); cost.initializeTL(dx, end, tstep); @@ -175,7 +163,7 @@ void LinearModel::forecastTL(Increment_ & dx, const ModelAuxIncr_ & mctl, } } else { while (dx.validTime() < end) { - this->stepTL(dx, mctl); + this->stepTL(dx, maux); cost.processTL(dx); post.process(dx); } @@ -192,7 +180,7 @@ void LinearModel::forecastTL(Increment_ & dx, const ModelAuxIncr_ & mctl, // ----------------------------------------------------------------------------- template -void LinearModel::forecastAD(Increment_ & dx, ModelAuxIncr_ & mctl, +void LinearModel::forecastAD(Increment_ & dx, ModelAuxInc_ & maux, const util::Duration & len, PostProcessor post, PostProcessorTLAD cost, @@ -200,7 +188,7 @@ void LinearModel::forecastAD(Increment_ & dx, ModelAuxIncr_ & mctl, Log::trace() << "LinearModel::forecastAD starting" << std::endl; const util::DateTime bgn(dx.validTime() - len); - const util::Duration tstep(tlm_->timeResolution()); + const util::Duration tstep(linearmodel_->timeResolution()); Log::info() << "LinearModel::forecastAD: Starting " << dx << std::endl; this->initializeAD(dx); post.initialize(dx, bgn, tstep); @@ -214,7 +202,7 @@ void LinearModel::forecastAD(Increment_ & dx, ModelAuxIncr_ & mctl, } else { while (dx.validTime() > bgn) { cost.processAD(dx); - this->stepAD(dx, mctl); + this->stepAD(dx, maux); post.process(dx); } } @@ -231,32 +219,21 @@ void LinearModel::forecastAD(Increment_ & dx, ModelAuxIncr_ & mctl, // ----------------------------------------------------------------------------- -template -void LinearModel::setTrajectory(const State_ & xx, State_ & xlr, - const ModelAux_ & maux) { - Log::trace() << "LinearModel::setTrajectory starting" << std::endl; - util::Timer timer(classname(), "setTrajectory"); - tlm_->setTrajectory(xx, xlr, maux); - Log::trace() << "LinearModel::setTrajectory done" << std::endl; -} - -// ----------------------------------------------------------------------------- - template void LinearModel::initializeTL(Increment_ & dx) const { Log::trace() << "LinearModel::initializeTL starting" << std::endl; util::Timer timer(classname(), "initializeTL"); - tlm_->initializeTL(dx); + linearmodel_->initializeTL(dx); Log::trace() << "LinearModel::initializeTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void LinearModel::stepTL(Increment_ & dx, const ModelAuxIncr_ & merr) const { +void LinearModel::stepTL(Increment_ & dx, const ModelAuxInc_ & maux) const { Log::trace() << "LinearModel::stepTL starting" << std::endl; util::Timer timer(classname(), "stepTL"); - tlm_->stepTL(dx, merr); + linearmodel_->stepTL(dx, maux); Log::trace() << "LinearModel::stepTL done" << std::endl; } @@ -266,7 +243,7 @@ template void LinearModel::finalizeTL(Increment_ & dx) const { Log::trace() << "LinearModel::finalizeTL starting" << std::endl; util::Timer timer(classname(), "finalizeTL"); - tlm_->finalizeTL(dx); + linearmodel_->finalizeTL(dx); Log::trace() << "LinearModel::finalizeTL done" << std::endl; } @@ -276,17 +253,17 @@ template void LinearModel::initializeAD(Increment_ & dx) const { Log::trace() << "LinearModel::initializeAD starting" << std::endl; util::Timer timer(classname(), "initializeAD"); - tlm_->initializeAD(dx); + linearmodel_->initializeAD(dx); Log::trace() << "LinearModel::initializeAD done" << std::endl; } // ----------------------------------------------------------------------------- template -void LinearModel::stepAD(Increment_ & dx, ModelAuxIncr_ & merr) const { +void LinearModel::stepAD(Increment_ & dx, ModelAuxInc_ & maux) const { Log::trace() << "LinearModel::stepAD starting" << std::endl; util::Timer timer(classname(), "stepAD"); - tlm_->stepAD(dx, merr); + linearmodel_->stepAD(dx, maux); Log::trace() << "LinearModel::stepAD done" << std::endl; } @@ -296,7 +273,7 @@ template void LinearModel::finalizeAD(Increment_ & dx) const { Log::trace() << "LinearModel::finalizeAD starting" << std::endl; util::Timer timer(classname(), "finalizeAD"); - tlm_->finalizeAD(dx); + linearmodel_->finalizeAD(dx); Log::trace() << "LinearModel::finalizeAD done" << std::endl; } @@ -306,12 +283,23 @@ template void LinearModel::print(std::ostream & os) const { Log::trace() << "LinearModel::print starting" << std::endl; util::Timer timer(classname(), "print"); - os << *tlm_; + os << *linearmodel_; Log::trace() << "LinearModel::print done" << std::endl; } // ----------------------------------------------------------------------------- +template +void LinearModel::setTrajectory(const State_ & xx, State_ & xtraj, + const ModelAuxCtl_ & maux) { + Log::trace() << "LinearModel::setTrajectory starting" << std::endl; + util::Timer timer(classname(), "setTrajectory"); + linearmodel_->setTrajectory(xx, xtraj, maux); + Log::trace() << "LinearModel::setTrajectory done" << std::endl; +} + +// ----------------------------------------------------------------------------- + } // namespace oops -#endif // OOPS_INTERFACE_LINEARMODEL_H_ +#endif // OOPS_BASE_LINEARMODEL_H_ diff --git a/src/oops/base/LinearModelBase.h b/src/oops/base/LinearModelBase.h deleted file mode 100644 index 9f369ef24..000000000 --- a/src/oops/base/LinearModelBase.h +++ /dev/null @@ -1,340 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_BASE_LINEARMODELBASE_H_ -#define OOPS_BASE_LINEARMODELBASE_H_ - -#include -#include -#include -#include - -#include -#include - -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/ModelAuxControl.h" -#include "oops/interface/ModelAuxIncrement.h" -#include "oops/interface/State.h" -#include "oops/util/AssociativeContainers.h" -#include "oops/util/Duration.h" -#include "oops/util/Logger.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/parameters/ConfigurationParameter.h" -#include "oops/util/parameters/HasParameters_.h" -#include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredPolymorphicParameter.h" -#include "oops/util/Printable.h" - -namespace eckit { - class Configuration; -} - -namespace oops { - -/// Base class for encapsulation of the linear forecast model. -/*! - * Defines the interfaces for the linear model. - */ - -// ----------------------------------------------------------------------------- - -template -class LinearModelBase : public util::Printable, - private boost::noncopyable { - typedef Increment Increment_; - typedef Geometry Geometry_; - typedef ModelAuxControl ModelAux_; - typedef ModelAuxIncrement ModelAuxIncr_; - typedef State State_; - - public: - static const std::string classname() {return "oops::LinearModelBase";} - - LinearModelBase() {} - virtual ~LinearModelBase() {} - -// Set the linearization trajectory - void setTrajectory(const State_ &, State_ &, const ModelAux_ &); - -// Run the TL forecast - void initializeTL(Increment_ &) const; - void stepTL(Increment_ &, const ModelAuxIncr_ &) const; - void finalizeTL(Increment_ &) const; - -// Run the AD forecast - void initializeAD(Increment_ &) const; - void stepAD(Increment_ &, ModelAuxIncr_ &) const; - void finalizeAD(Increment_ &) const; - -// Information and diagnostics - virtual const util::Duration & timeResolution() const = 0; - virtual const oops::Variables & variables() const = 0; - - protected: -// Set the linearization trajectory - virtual void setTrajectory(const typename MODEL::State &, typename MODEL::State &, - const typename MODEL::ModelAuxControl &) = 0; - -// Run the TL forecast - virtual void initializeTL(typename MODEL::Increment &) const = 0; - virtual void stepTL(typename MODEL::Increment &, - const typename MODEL::ModelAuxIncrement &) const = 0; - virtual void finalizeTL(typename MODEL::Increment &) const = 0; - -// Run the AD forecast - virtual void initializeAD(typename MODEL::Increment &) const = 0; - virtual void stepAD(typename MODEL::Increment &, typename MODEL::ModelAuxIncrement &) const = 0; - virtual void finalizeAD(typename MODEL::Increment &) const = 0; - -// Information and diagnostics - virtual void print(std::ostream &) const = 0; -}; - -// ============================================================================= - -template -class LinearModelFactory; - -// ----------------------------------------------------------------------------- - -/// \brief Base class for classes storing model-specific parameters. -class LinearModelParametersBase : public Parameters { - OOPS_ABSTRACT_PARAMETERS(LinearModelParametersBase, Parameters) - public: - /// \brief Model name. - /// - /// \note This parameter is marked as optional because it is only required in certain - /// circumstances (e.g. when model parameters are deserialized into a - /// LinearModelParametersWrapper and used by LinearModelFactory to instantiate a tangent linear - /// model whose type is determined at runtime), but not others (e.g. in tests written with a - /// particular model in mind). LinearModelParametersWrapper will throw an exception if this - /// parameter is not provided. - OptionalParameter name{"name", this}; -}; - -// ----------------------------------------------------------------------------- - -/// \brief A subclass of LinearModelParametersBase storing the values of all options in a -/// single Configuration object. -/// -/// This object can be accessed by calling the value() method of the \p config member variable. -/// -/// The ConfigurationParameter class does not perform any parameter validation; models using -/// GenericLinearModelParameters should therefore ideally be refactored, replacing this class with a -/// dedicated subclass of LinearModelParametersBase storing each parameter in a separate -/// (Optional/Required)Parameter object. -class GenericLinearModelParameters : public LinearModelParametersBase { - OOPS_CONCRETE_PARAMETERS(GenericLinearModelParameters, LinearModelParametersBase) - public: - ConfigurationParameter config{this}; -}; - -// ----------------------------------------------------------------------------- - -/// \brief Contains a polymorphic parameter holding an instance of a subclass of -/// LinearModelParametersBase. -template -class LinearModelParametersWrapper : public Parameters { - OOPS_CONCRETE_PARAMETERS(LinearModelParametersWrapper, Parameters) - public: - /// After deserialization, holds an instance of a subclass of LinearModelParametersBase - /// controlling the behavior of a tangent linear model. The type of the subclass is determined by - /// the value of the "name" key in the Configuration object from which this object is - /// deserialized. - RequiredPolymorphicParameter> - modelParameters{"name", this}; -}; - -// ============================================================================= - -/// \brief Tangent linear model factory. -template -class LinearModelFactory { - typedef Geometry Geometry_; - - public: - /// \brief Create and return a new tangent linear model. - /// - /// The model's type is determined by the \c name attribute of \p parameters. - /// \p parameters must be an instance of the subclass of LinearModelParametersBase - /// associated with that model type, otherwise an exception will be thrown. - static LinearModelBase * create(const Geometry_ &, - const LinearModelParametersBase & parameters); - - /// \brief Create and return an instance of the subclass of LinearModelParametersBase - /// storing parameters of tangent linear models of the specified type. - static std::unique_ptr createParameters(const std::string &name); - - /// \brief Return the names of all tangent linear models that can be created by one of the - /// registered makers. - static std::vector getMakerNames() { - return keys(getMakers()); - } - - virtual ~LinearModelFactory() = default; - - protected: - /// \brief Register a maker able to create tangent linear models of type \p name. - explicit LinearModelFactory(const std::string &name); - - private: - virtual LinearModelBase * make(const Geometry_ &, const LinearModelParametersBase &) = 0; - - virtual std::unique_ptr makeParameters() const = 0; - - static std::map < std::string, LinearModelFactory * > & getMakers() { - static std::map < std::string, LinearModelFactory * > makers_; - return makers_; - } -}; - -// ----------------------------------------------------------------------------- - -template -class LinearModelMaker : public LinearModelFactory { - private: - /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as - /// GenericLinearModelParameters. - typedef TParameters_IfAvailableElseFallbackType_t Parameters_; - - typedef Geometry Geometry_; - - LinearModelBase * make(const Geometry_ & geom, - const LinearModelParametersBase & parameters) override { - const auto &stronglyTypedParameters = dynamic_cast(parameters); - return new T(geom.geometry(), - parametersOrConfiguration::value>(stronglyTypedParameters)); - } - - std::unique_ptr makeParameters() const override { - return boost::make_unique(); - } - - public: - explicit LinearModelMaker(const std::string & name) : LinearModelFactory(name) {} -}; - -// ----------------------------------------------------------------------------- - -template -LinearModelFactory::LinearModelFactory(const std::string & name) { - if (getMakers().find(name) != getMakers().end()) { - throw std::runtime_error(name + " already registered in tangent linear model factory."); - } - getMakers()[name] = this; -} - -// ----------------------------------------------------------------------------- - -template -LinearModelBase* LinearModelFactory::create( - const Geometry_ & geom, const LinearModelParametersBase & parameters) { - Log::trace() << "LinearModelBase::create starting" << std::endl; - const std::string &id = parameters.name.value().value(); - typename std::map*>::iterator - jerr = getMakers().find(id); - if (jerr == getMakers().end()) { - Log::error() << id << " does not exist in the tangent linear model factory." << std::endl; - Log::error() << "Factory contains " << getMakers().size() << " elements:" << std::endl; - for (typename std::map*>::const_iterator - jj = getMakers().begin(); jj != getMakers().end(); ++jj) { - Log::error() << "A " << jj->first << " linear model" << std::endl; - } - throw std::runtime_error(id + " does not exist in tangent linear model factory."); - } - LinearModelBase * ptr = jerr->second->make(geom, parameters); - Log::trace() << "LinearModelBase::create done" << std::endl; - return ptr; -} - -// ----------------------------------------------------------------------------- - -template -std::unique_ptr LinearModelFactory::createParameters( - const std::string &name) { - typename std::map*>::iterator it = getMakers().find(name); - if (it == getMakers().end()) { - throw std::runtime_error(name + " does not exist in the tangent linear model factory"); - } - return it->second->makeParameters(); -} - -// ============================================================================= - -template -void LinearModelBase::setTrajectory(const State_ & xx, State_ & xlr, - const ModelAux_ & maux) { - Log::trace() << "LinearModelBase::setTrajectory starting" << std::endl; - this->setTrajectory(xx.state(), xlr.state(), maux.modelauxcontrol()); - Log::trace() << "LinearModelBase::setTrajectory done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::initializeTL(Increment_ & dx) const { - Log::trace() << "LinearModelBase::initializeTL starting" << std::endl; - this->initializeTL(dx.increment()); - Log::trace() << "LinearModelBase::initializeTL done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::stepTL(Increment_ & dx, const ModelAuxIncr_ & merr) const { - Log::trace() << "LinearModelBase::stepTL starting" << std::endl; - this->stepTL(dx.increment(), merr.modelauxincrement()); - Log::trace() << "LinearModelBase::stepTL done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::finalizeTL(Increment_ & dx) const { - Log::trace() << "LinearModelBase::finalizeTL starting" << std::endl; - this->finalizeTL(dx.increment()); - Log::trace() << "LinearModelBase::finalizeTL done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::initializeAD(Increment_ & dx) const { - Log::trace() << "LinearModelBase::initializeAD starting" << std::endl; - this->initializeAD(dx.increment()); - Log::trace() << "LinearModelBase::initializeAD done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::stepAD(Increment_ & dx, ModelAuxIncr_ & merr) const { - Log::trace() << "LinearModelBase::stepAD starting" << std::endl; - this->stepAD(dx.increment(), merr.modelauxincrement()); - Log::trace() << "LinearModelBase::stepAD done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelBase::finalizeAD(Increment_ & dx) const { - Log::trace() << "LinearModelBase::finalizeAD starting" << std::endl; - this->finalizeAD(dx.increment()); - Log::trace() << "LinearModelBase::finalizeAD done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_BASE_LINEARMODELBASE_H_ diff --git a/src/oops/base/LinearVariableChangeBase.h b/src/oops/base/LinearVariableChangeBase.h index 10c1cab0b..253dd62dc 100644 --- a/src/oops/base/LinearVariableChangeBase.h +++ b/src/oops/base/LinearVariableChangeBase.h @@ -18,11 +18,11 @@ #include "eckit/exception/Exceptions.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeParametersBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/AssociativeContainers.h" #include "oops/util/parameters/ConfigurationParameter.h" #include "oops/util/parameters/HasParameters_.h" diff --git a/src/oops/base/Localization.h b/src/oops/base/Localization.h new file mode 100644 index 000000000..e9dfcf367 --- /dev/null +++ b/src/oops/base/Localization.h @@ -0,0 +1,189 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_BASE_LOCALIZATION_H_ +#define OOPS_BASE_LOCALIZATION_H_ + +#include +#include + +#include + +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/generic/LocalizationBase.h" +#include "oops/mpi/mpi.h" +#include "oops/util/Logger.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" +#include "oops/util/Timer.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +/// \brief Abstract model-space localization class used by high level algorithms +/// and applications. +/// +/// Note: to see methods that need to be implemented in a generic Localization +/// implementation, see LocalizationBase class in generic/LocalizationBase.h. +/// To see methods that need to be implemented in a MODEL-specific Localization +/// implementation, see interface::LocalizationBase class in +/// interface/LocalizationBase.h. +template +class Localization : public util::Printable, + private boost::noncopyable, + private util::ObjectCounter > { + typedef Geometry Geometry_; + typedef Increment Increment_; + typedef LocalizationBase LocBase_; + + public: + static const std::string classname() {return "oops::Localization";} + + /// Set up Localization for \p geometry, configured with \p conf + Localization(const Geometry_ & geometry, + const eckit::Configuration & conf); + ~Localization(); + + /// Randomize \p dx and apply 4D localization. All 3D blocks of the 4D localization + /// matrix are the same (and defined by 3D localization loc_) + virtual void randomize(Increment_ & dx) const; + /// Apply 4D localization. All 3D blocks of the 4D localization matrix are the same + /// (and defined by 3D localization loc_) + virtual void multiply(Increment_ & dx) const; + + private: + /// Print, used in logging + void print(std::ostream &) const override; + + /// Pointer to the Localization implementation + std::unique_ptr loc_; +}; + +// ----------------------------------------------------------------------------- + +template +Localization::Localization(const Geometry_ & geometry, + const eckit::Configuration & conf) + : loc_(LocalizationFactory::create(geometry, conf)) +{ + Log::trace() << "Localization::Localization done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +Localization::~Localization() { + Log::trace() << "Localization::~Localization starting" << std::endl; + util::Timer timer(classname(), "~Localization"); + loc_.reset(); + Log::trace() << "Localization::~Localization done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void Localization::randomize(Increment_ & dx) const { + Log::trace() << "Localization::randomize starting" << std::endl; + util::Timer timer(classname(), "randomize"); + const eckit::mpi::Comm & comm = dx.timeComm(); + static int tag = 23456; + size_t nslots = comm.size(); + int mytime = comm.rank(); + + if (mytime > 0) { + util::DateTime dt = dx.validTime(); // Save original time value + dx.zero(); + oops::mpi::receive(comm, dx, 0, tag); + dx.updateTime(dt - dx.validTime()); // Set time back to original value + } else { + // Apply 3D localization + loc_->randomize(dx); + + // Copy result to all timeslots + for (size_t jj = 1; jj < nslots; ++jj) { + oops::mpi::send(comm, dx, jj, tag); + } + } + ++tag; + Log::trace() << "Localization::randomize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void Localization::multiply(Increment_ & dx) const { + Log::trace() << "Localization::multiply starting" << std::endl; + util::Timer timer(classname(), "multiply"); + const eckit::mpi::Comm & comm = dx.timeComm(); + static int tag = 23456; + size_t nslots = comm.size(); + int mytime = comm.rank(); + + // Use Mark Buehner's trick to save CPU when applying the same 3D localization for all + // 3D blocks of the 4D localization matrix: + // L_4D = ( L_3D L_3D L_3D ) = ( Id ) L_3D ( Id Id Id ) + // ( L_3D L_3D L_3D ) ( Id ) + // ( L_3D L_3D L_3D ) ( Id ) + // so if : + // x_4D = ( x_1 ) + // ( x_2 ) + // ( x_3 ) + // then: + // L_4D x_4D = (Id) L_3D (Id Id Id) (x_1) = (Id) L_3D (x_1+x_2+x_3) = (L_3D ( x_1 + x_2 + x_3 )) + // (Id) (x_2) (Id) (L_3D ( x_1 + x_2 + x_3 )) + // (Id) (x_3) (Id) (L_3D ( x_1 + x_2 + x_3 )) + // Reference in section 3.4.2. of https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/qj.2325. + if (mytime > 0) { + util::DateTime dt = dx.validTime(); // Save original time value + oops::mpi::send(comm, dx, 0, tag); + dx.zero(); + oops::mpi::receive(comm, dx, 0, tag); + dx.updateTime(dt - dx.validTime()); // Set time back to original value + } else { + // Sum over timeslots + for (size_t jj = 1; jj < nslots; ++jj) { + Increment_ dxtmp(dx); + oops::mpi::receive(comm, dxtmp, jj, tag); + dx.axpy(1.0, dxtmp, false); + } + + // Apply 3D localization + loc_->multiply(dx); + + // Copy result to all timeslots + for (size_t jj = 1; jj < nslots; ++jj) { + oops::mpi::send(comm, dx, jj, tag); + } + } + ++tag; + + Log::trace() << "Localization::multiply done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void Localization::print(std::ostream & os) const { + Log::trace() << "Localization::print starting" << std::endl; + util::Timer timer(classname(), "print"); + os << *loc_; + Log::trace() << "Localization::print done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_LOCALIZATION_H_ diff --git a/src/oops/interface/Model.h b/src/oops/base/Model.h similarity index 83% rename from src/oops/interface/Model.h rename to src/oops/base/Model.h index 80c6b1b3e..0ac3460ad 100644 --- a/src/oops/interface/Model.h +++ b/src/oops/base/Model.h @@ -6,18 +6,19 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_INTERFACE_MODEL_H_ -#define OOPS_INTERFACE_MODEL_H_ +#ifndef OOPS_BASE_MODEL_H_ +#define OOPS_BASE_MODEL_H_ #include #include +#include #include -#include "oops/base/ModelBase.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" +#include "oops/generic/ModelBase.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" @@ -29,30 +30,20 @@ namespace eckit { namespace oops { -/// \brief Encapsulates the nonlinear forecast model -/// Note: to see methods that need to be implemented in the forecast model implementation, -/// see ModelBase class. +/// \brief Abstract nonlinear forecast model used by high level algorithms and applications. /// -/// Note: implementations of this interface can opt to extract their settings either from -/// a Configuration object or from a subclass of ModelParametersBase. +/// Note: to see methods that need to be implemented in a generic forecast model +/// implementation, see ModelBase class in generic/ModelBase.h. To see methods that need +/// to be implemented in a MODEL-specific forecast model implementation, see +/// interface::ModelBase class in interface/ModelBase.h. /// -/// In the former case, they should provide a constructor with the following signature: -/// -/// Model(const Geometry_ &, const eckit::Configuration &); -/// -/// In the latter case, the implementer should first define a subclass of ModelParametersBase -/// holding the settings of the model in question. The implementation of the Model interface -/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with -/// the following signature: -/// -/// Model(const Geometry_ &, const Parameters_ &); // ----------------------------------------------------------------------------- template class Model : public util::Printable, private boost::noncopyable, private util::ObjectCounter > { - typedef GenericModelBase ModelBase_; + typedef ModelBase ModelBase_; typedef Geometry Geometry_; typedef ModelAuxControl ModelAux_; typedef State State_; @@ -62,6 +53,7 @@ class Model : public util::Printable, Model(const Geometry_ &, const ModelParametersBase &); Model(const Geometry_ &, const eckit::Configuration &); + explicit Model(std::unique_ptr); virtual ~Model(); /// \brief Run the forecast from state \p xx for \p len time, with \p post postprocessors @@ -111,6 +103,15 @@ Model::Model(const Geometry_ & resol, const eckit::Configuration & conf) // ----------------------------------------------------------------------------- +template +Model::Model(std::unique_ptr model) + : model_(std::move(model)) +{ + Log::trace() << "Model::Model created" << std::endl; +} + +// ----------------------------------------------------------------------------- + template Model::~Model() { Log::trace() << "Model::~Model starting" << std::endl; @@ -188,4 +189,4 @@ void Model::print(std::ostream & os) const { } // namespace oops -#endif // OOPS_INTERFACE_MODEL_H_ +#endif // OOPS_BASE_MODEL_H_ diff --git a/src/oops/base/ModelSpaceCovarianceBase.h b/src/oops/base/ModelSpaceCovarianceBase.h index 7aba9e190..5179c1d02 100644 --- a/src/oops/base/ModelSpaceCovarianceBase.h +++ b/src/oops/base/ModelSpaceCovarianceBase.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -23,12 +23,14 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" +#include "oops/assimilation/GMRESR.h" +#include "oops/base/Geometry.h" +#include "oops/base/IdentityMatrix.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" #include "oops/base/ModelSpaceCovarianceParametersBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/AssociativeContainers.h" #include "oops/util/Logger.h" #include "oops/util/parameters/ConfigurationParameter.h" @@ -88,6 +90,9 @@ class ModelSpaceCovarianceBase { ChvarVec_ chvars_; size_t randomizationSize_; + bool fullInverse_; + int fullInverseIterations_; + double fullInverseAccuracy_; }; // ============================================================================= @@ -306,6 +311,9 @@ ModelSpaceCovarianceBase::ModelSpaceCovarianceBase( bg, fg, resol, variableChange.variableChangeParameters)); } randomizationSize_ = parameters.randomizationSize; + fullInverse_ = parameters.fullInverse; + fullInverseIterations_ = parameters.fullInverseIterations; + fullInverseAccuracy_ = parameters.fullInverseAccuracy; } // ----------------------------------------------------------------------------- @@ -368,26 +376,33 @@ void ModelSpaceCovarianceBase::multiply(const Increment_ & dxi, template void ModelSpaceCovarianceBase::inverseMultiply(const Increment_ & dxi, Increment_ & dxo) const { - if (chvars_.size()) { - // K_1^{-1} K_2^{-1} .. K_N^{-1} - std::unique_ptr dxchvarin(new Increment_(dxi)); - for (ircst_ it = chvars_.rbegin(); it != chvars_.rend(); ++it) { - Increment_ dxchvarout = it->multiplyInverse(*dxchvarin); - dxchvarin.reset(new Increment_(dxchvarout)); - } - Increment_ dxchvarout(*dxchvarin, false); + if (fullInverse_) { + // Approximate full inverse using GMRESR + IdentityMatrix Id; + dxo.zero(); + GMRESR(dxo, dxi, *this, Id, fullInverseIterations_, fullInverseAccuracy_); + } else { + if (chvars_.size()) { + // K_1^{-1} K_2^{-1} .. K_N^{-1} + std::unique_ptr dxchvarin(new Increment_(dxi)); + for (ircst_ it = chvars_.rbegin(); it != chvars_.rend(); ++it) { + Increment_ dxchvarout = it->multiplyInverse(*dxchvarin); + dxchvarin.reset(new Increment_(dxchvarout)); + } + Increment_ dxchvarout(*dxchvarin, false); - this->doInverseMultiply(*dxchvarin, dxchvarout); + this->doInverseMultiply(*dxchvarin, dxchvarout); - // K_N^T^{-1} K_N-1^T^{-1} ... K_1^T^{-1} - dxchvarin.reset(new Increment_(dxchvarout)); - for (icst_ it = chvars_.begin(); it != chvars_.end(); ++it) { - Increment_ dxchvarout = it->multiplyInverseAD(*dxchvarin); + // K_N^T^{-1} K_N-1^T^{-1} ... K_1^T^{-1} dxchvarin.reset(new Increment_(dxchvarout)); + for (icst_ it = chvars_.begin(); it != chvars_.end(); ++it) { + Increment_ dxchvarout = it->multiplyInverseAD(*dxchvarin); + dxchvarin.reset(new Increment_(dxchvarout)); + } + dxo = *dxchvarin; + } else { + this->doInverseMultiply(dxi, dxo); } - dxo = *dxchvarin; - } else { - this->doInverseMultiply(dxi, dxo); } } diff --git a/src/oops/base/ModelSpaceCovarianceParametersBase.h b/src/oops/base/ModelSpaceCovarianceParametersBase.h index cf481b542..d5eea7bfa 100644 --- a/src/oops/base/ModelSpaceCovarianceParametersBase.h +++ b/src/oops/base/ModelSpaceCovarianceParametersBase.h @@ -38,6 +38,9 @@ class ModelSpaceCovarianceParametersBase : public Parameters { Parameter>> variableChanges{ "variable changes", {}, this}; Parameter randomizationSize{"randomization size", 50, this}; + Parameter fullInverse{"full inverse", false, this}; + Parameter fullInverseIterations{"full inverse iterations", 10, this}; + Parameter fullInverseAccuracy{"full inverse accuracy", 1.0e-3, this}; }; } // namespace oops diff --git a/src/oops/base/ObsAuxControls.h b/src/oops/base/ObsAuxControls.h index a2f90fedd..167994db0 100644 --- a/src/oops/base/ObsAuxControls.h +++ b/src/oops/base/ObsAuxControls.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2017-2019 UCAR - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ @@ -15,11 +15,15 @@ #include "oops/base/ObsSpaces.h" #include "oops/interface/ObsAuxControl.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" +#include "oops/util/parameters/Parameters.h" #include "oops/util/Printable.h" namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Holds a vector of ObsAuxControl // ----------------------------------------------------------------------------- template @@ -32,6 +36,7 @@ class ObsAuxControls : public util::Printable { static const std::string classname() {return "oops::ObsAuxControls";} ObsAuxControls(const ObsSpaces_ &, const eckit::Configuration &); + ObsAuxControls(const ObsSpaces_ &, const std::vector &); explicit ObsAuxControls(const ObsAuxControls &, const bool copy = true); ~ObsAuxControls(); @@ -55,21 +60,29 @@ class ObsAuxControls : public util::Printable { // ============================================================================= template -ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, const eckit::Configuration & conf) +ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, + const std::vector & params) : auxs_(0) { - std::vector obsconf = conf.getSubConfigurations(); - for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { - eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); - Parameters_ obsauxparams; - obsauxparams.validateAndDeserialize(obsauxconf); + ASSERT(odb.size() == params.size()); + for (std::size_t jobs = 0; jobs < params.size(); ++jobs) { auxs_.push_back( - std::unique_ptr(new ObsAuxControl_(odb[jobs], obsauxparams))); + std::unique_ptr(new ObsAuxControl_(odb[jobs], params[jobs]))); } } // ----------------------------------------------------------------------------- +template +ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, const eckit::Configuration & conf) + : ObsAuxControls(odb, + // Split conf into subconfigurations, extract the "obs bias" section from each + // of them, then validate and deserialize that section into a Parameters_ object + validateAndDeserialize(util::vectoriseAndFilter(conf, "obs bias"))) +{} + +// ----------------------------------------------------------------------------- + template ObsAuxControls::ObsAuxControls(const ObsAuxControls & other, const bool copy) : auxs_(other.size()) @@ -145,7 +158,9 @@ double ObsAuxControls::norm() const { template void ObsAuxControls::print(std::ostream & os) const { - for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) os << *auxs_[jobs]; + for (const auto & aux : auxs_) { + os << *aux << std::endl; + } } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsAuxCovariances.h b/src/oops/base/ObsAuxCovariances.h index afd36ae5b..0d11b643c 100644 --- a/src/oops/base/ObsAuxCovariances.h +++ b/src/oops/base/ObsAuxCovariances.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2017-2019 UCAR - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ @@ -25,6 +25,8 @@ namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Holds a vector of ObsAuxCovariance // ----------------------------------------------------------------------------- template diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index 6b7f944d1..ed758cbd4 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -23,6 +23,8 @@ namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Holds a vector of ObsAuxIncrement // ----------------------------------------------------------------------------- template diff --git a/src/oops/base/ObsError.h b/src/oops/base/ObsError.h new file mode 100644 index 000000000..54367d48a --- /dev/null +++ b/src/oops/base/ObsError.h @@ -0,0 +1,197 @@ +/* + * (C) Crown copyright 2021, Met Office. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_OBSERROR_H_ +#define OOPS_BASE_OBSERROR_H_ + +#include +#include + +#include +#include "eckit/config/Configuration.h" + +#include "oops/base/ObsVector.h" +#include "oops/generic/ObsErrorBase.h" +#include "oops/interface/ObsSpace.h" +#include "oops/util/Logger.h" +#include "oops/util/Printable.h" + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief Observation error covariance matrix of observations from a single ObsSpace. +template +class ObsError : public util::Printable, + private util::ObjectCounter > { + typedef ObsErrorBase ObsErrorBase_; + typedef ObsVector ObsVector_; + typedef ObsSpace ObsSpace_; + + public: + static const std::string classname() {return "oops::ObsError";} + + ObsError(const ObsErrorParametersBase & params, const ObsSpace_ & os); + ~ObsError() override; + ObsError(const ObsError &) = delete; + ObsError(ObsError &&) = default; + ObsError& operator=(const ObsError &) = delete; + ObsError& operator=(ObsError &&) = default; + + /// Multiply a Departure \p dy by \f$R\f$. + void multiply(ObsVector_ & dy) const; + /// Multiply a Departure \p dy by \f$R^{-1}\f$. + void inverseMultiply(ObsVector_ & dy) const; + + /// Generate a random perturbation in \p dy. + void randomize(ObsVector_ & dy) const; + + /// Save obs errors. + void save(const std::string &) const; + + /// Return a copy of obs error std. dev. If this ObsVector_ is modified (e.g. by obs filters), + /// it should be passed back to update() to ensure the covariance matrix stays consistent. + ObsVector_ obserrors() const; + + /// Set the diagonal of the covariance matrix to \p stddev squared. + void update(const ObsVector_ &stddev); + + /// Return the vector of inverse obs error variances. + ObsVector_ inverseVariance() const; + + /// Get mean error for Jo table. + double getRMSE() const; + + private: + void print(std::ostream &) const override; + + private: + std::unique_ptr err_; +}; + +// ----------------------------------------------------------------------------- + +template +ObsError::ObsError(const ObsErrorParametersBase & params, const ObsSpace_ & os) { + Log::trace() << "ObsError::ObsError starting" << std::endl; + + util::Timer timer(classname(), "ObsErrors"); + size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); + + err_ = ObsErrorFactory::create(params, os); + + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); + this->setObjectSize(current - init); + Log::trace() << "ObsError::ObsError done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +ObsError::~ObsError() { + Log::trace() << "ObsError::~ObsError starting" << std::endl; + util::Timer timer(classname(), "~ObsError"); + err_.reset(); + Log::trace() << "ObsError::~ObsError done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsError::multiply(ObsVector_ & dy) const { + Log::trace() << "ObsError::multiply starting" << std::endl; + util::Timer timer(classname(), "multiply"); + err_->multiply(dy); + Log::trace() << "ObsError::multiply done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsError::inverseMultiply(ObsVector_ & dy) const { + Log::trace() << "ObsError::inverseMultiply starting" << std::endl; + util::Timer timer(classname(), "inverseMultiply"); + err_->inverseMultiply(dy); + Log::trace() << "ObsError::inverseMultiply done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsError::randomize(ObsVector_ & dy) const { + Log::trace() << "ObsError::randomize starting" << std::endl; + util::Timer timer(classname(), "randomize"); + err_->randomize(dy); + Log::trace() << "ObsError::randomize done" << std::endl; +} + +// ----------------------------------------------------------------------------- +template +void ObsError::save(const std::string & name) const { + Log::trace() << "ObsError::save starting" << std::endl; + util::Timer timer(classname(), "save"); + err_->save(name); + Log::trace() << "ObsError::save done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +typename ObsError::ObsVector_ ObsError::obserrors() const { + Log::trace() << "ObsError::obserrors starting" << std::endl; + util::Timer timer(classname(), "obserrors"); + ObsVector_ obserr = err_->obserrors(); + Log::trace() << "ObsError::obserrors done" << std::endl; + return obserr; +} + +// ----------------------------------------------------------------------------- + +template +void ObsError::update(const ObsVector_ & obserr) { + Log::trace() << "ObsError::update starting" << std::endl; + util::Timer timer(classname(), "update"); + err_->update(obserr); + Log::trace() << "ObsError::update done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +ObsVector ObsError::inverseVariance() const { + Log::trace() << "ObsError::inverseVariance starting" << std::endl; + util::Timer timer(classname(), "inverseVariance"); + ObsVector_ invar = err_->inverseVariance(); + Log::trace() << "ObsError::inverseVariance done" << std::endl; + return invar; +} + +// ----------------------------------------------------------------------------- + +template +double ObsError::getRMSE() const { + Log::trace() << "ObsError::getRMSE starting" << std::endl; + util::Timer timer(classname(), "getRMSE"); + double zz = err_->getRMSE(); + Log::trace() << "ObsError::getRMSE done" << std::endl; + return zz; +} + +// ----------------------------------------------------------------------------- + +template +void ObsError::print(std::ostream & os) const { + Log::trace() << "ObsError::print starting" << std::endl; + util::Timer timer(classname(), "print"); + os << (*err_); + Log::trace() << "ObsError::print done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSERROR_H_ diff --git a/src/oops/base/ObsErrorBase.h b/src/oops/base/ObsErrorBase.h deleted file mode 100644 index 7963fbd3d..000000000 --- a/src/oops/base/ObsErrorBase.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_BASE_OBSERRORBASE_H_ -#define OOPS_BASE_OBSERRORBASE_H_ - -#include -#include -#include - -#include -#include "eckit/config/Configuration.h" - -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/util/abor1_cpp.h" -#include "oops/util/Logger.h" -#include "oops/util/Printable.h" - -namespace oops { - -// ----------------------------------------------------------------------------- -/// \brief Base class for observation error covariance matrices. -template -class ObsErrorBase : public util::Printable, - private boost::noncopyable { - typedef ObsVector ObsVector_; - typedef ObsSpace ObsSpace_; - - public: - ObsErrorBase() = default; - virtual ~ObsErrorBase() = default; - -/// Multiply a Departure \p dy by \f$R\f$$ - virtual void multiply(ObsVector_ & dy) const = 0; -/// Multiply a Departure \p dy by \f$R^{-1}\f$ - virtual void inverseMultiply(ObsVector_ & dy) const = 0; - -/// Generate random perturbation in \p dy - virtual void randomize(ObsVector_ & dy) const = 0; - -/// Save obs errors - virtual void save(const std::string &) const = 0; - -/// Return obs error std. dev. The non-const method means caller can modify values, -/// this is used by obs filters. In this case update() should be called to ensure -/// the matrix stays consistent. - virtual ObsVector_ & obserrors() = 0; - virtual const ObsVector_ & obserrors() const = 0; - -/// Update when obs errors standard deviations have been modified - virtual void update() = 0; - -/// Return inverseVariance - virtual const ObsVector_ & inverseVariance() const = 0; - -/// Get mean error for Jo table - virtual double getRMSE() const = 0; -}; - -// ============================================================================= - -/// ObsErrorFactory Factory -template -class ObsErrorFactory { - typedef ObsSpace ObsSpace_; - public: - static std::unique_ptr > create(const eckit::Configuration &, - const ObsSpace_ &); - virtual ~ObsErrorFactory() = default; - protected: - explicit ObsErrorFactory(const std::string &); - private: - virtual ObsErrorBase * make(const eckit::Configuration &, const ObsSpace_ &) = 0; - static std::map < std::string, ObsErrorFactory * > & getMakers() { - static std::map < std::string, ObsErrorFactory * > makers_; - return makers_; - } -}; - -// ----------------------------------------------------------------------------- - -template -class ObsErrorMaker : public ObsErrorFactory { - typedef ObsSpace ObsSpace_; - virtual ObsErrorBase * make(const eckit::Configuration & conf, const ObsSpace_ & obs) - { return new T(conf, obs); } - public: - explicit ObsErrorMaker(const std::string & name) : ObsErrorFactory(name) {} -}; - -// ============================================================================= - -template -ObsErrorFactory::ObsErrorFactory(const std::string & name) { - if (getMakers().find(name) != getMakers().end()) { - throw std::runtime_error(name + " already registered in obs error factory."); - } - getMakers()[name] = this; -} - -// ----------------------------------------------------------------------------- - -template -std::unique_ptr> -ObsErrorFactory::create(const eckit::Configuration & conf, const ObsSpace_ & obs) { - Log::trace() << "ObsErrorBase::create starting" << std::endl; - const std::string id = conf.getString("covariance model", "diagonal"); - typename std::map*>::iterator - jerr = getMakers().find(id); - if (jerr == getMakers().end()) { - throw std::runtime_error(id + " does not exist in obs error factory."); - } - std::unique_ptr> ptr(jerr->second->make(conf, obs)); - Log::trace() << "ObsErrorBase::create done" << std::endl; - return ptr; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_BASE_OBSERRORBASE_H_ diff --git a/src/oops/base/ObsErrors.h b/src/oops/base/ObsErrors.h index 2ee7e950a..add8dad5b 100644 --- a/src/oops/base/ObsErrors.h +++ b/src/oops/base/ObsErrors.h @@ -18,31 +18,35 @@ #include #include "oops/base/Departures.h" -#include "oops/base/ObsErrorBase.h" +#include "oops/base/ObsError.h" #include "oops/base/ObsSpaces.h" +#include "oops/util/ConfigFunctions.h" // for vectoriseAndFilter #include "oops/util/Logger.h" +#include "oops/util/parameters/Parameters.h" #include "oops/util/Printable.h" namespace oops { // ----------------------------------------------------------------------------- -/// \biref Container for ObsErrors for all observation types that are used in DA +/// \brief Container for ObsErrors for all observation types that are used in DA template class ObsErrors : public util::Printable, private boost::noncopyable { - typedef Departures Departures_; - typedef ObsErrorBase ObsError_; - typedef ObsSpaces ObsSpaces_; + typedef Departures Departures_; + typedef ObsError ObsError_; + typedef ObsErrorParametersWrapper Parameters_; + typedef ObsSpaces ObsSpaces_; public: static const std::string classname() {return "oops::ObsErrors";} + ObsErrors(const std::vector &, const ObsSpaces_ &); ObsErrors(const eckit::Configuration &, const ObsSpaces_ &); /// Accessor and size size_t size() const {return err_.size();} - ObsError_ & operator[](const size_t ii) {return *err_.at(ii);} - const ObsError_ & operator[](const size_t ii) const {return *err_.at(ii);} + ObsError_ & operator[](const size_t ii) {return err_.at(ii);} + const ObsError_ & operator[](const size_t ii) const {return err_.at(ii);} /// Multiply a Departure by \f$R\f$ void multiply(Departures_ &) const; @@ -52,33 +56,49 @@ class ObsErrors : public util::Printable, /// Generate random perturbation void randomize(Departures_ &) const; +/// Save obs errors + void save(const std::string &) const; + /// returns inverse of observation error variance Departures_ inverseVariance() const; private: - void print(std::ostream &) const; - std::vector > err_; + void print(std::ostream &) const override; + std::vector err_; const ObsSpaces_ & os_; }; // ----------------------------------------------------------------------------- template -ObsErrors::ObsErrors(const eckit::Configuration & config, +ObsErrors::ObsErrors(const std::vector & params, const ObsSpaces_ & os) : err_(), os_(os) { - std::vector obsconf = config.getSubConfigurations(); + ASSERT(params.empty() || params.size() == os.size()); + const Parameters_ defaultParam; + + err_.reserve(os.size()); for (size_t jj = 0; jj < os.size(); ++jj) { - eckit::LocalConfiguration conf = obsconf[jj].getSubConfiguration("obs error"); - err_.emplace_back(ObsErrorFactory::create(conf, os[jj])); + const Parameters_ & param = params.empty() ? defaultParam : params[jj]; + err_.emplace_back(param.obsErrorParameters, os_[jj]); } } // ----------------------------------------------------------------------------- +template +ObsErrors::ObsErrors(const eckit::Configuration & config, + const ObsSpaces_ & os) : + ObsErrors( // Split config into subconfigurations, extract the "obs error" section from each + // of them, then validate and deserialize that section into a Parameters_ object + validateAndDeserialize(util::vectoriseAndFilter(config, "obs error")), os) +{} + +// ----------------------------------------------------------------------------- + template void ObsErrors::multiply(Departures_ & dy) const { for (size_t jj = 0; jj < err_.size(); ++jj) { - err_[jj]->multiply(dy[jj]); + err_[jj].multiply(dy[jj]); } } @@ -87,7 +107,7 @@ void ObsErrors::multiply(Departures_ & dy) const { template void ObsErrors::inverseMultiply(Departures_ & dy) const { for (size_t jj = 0; jj < err_.size(); ++jj) { - err_[jj]->inverseMultiply(dy[jj]); + err_[jj].inverseMultiply(dy[jj]); } } @@ -96,7 +116,16 @@ void ObsErrors::inverseMultiply(Departures_ & dy) const { template void ObsErrors::randomize(Departures_ & dy) const { for (size_t jj = 0; jj < err_.size(); ++jj) { - err_[jj]->randomize(dy[jj]); + err_[jj].randomize(dy[jj]); + } +} + +// ----------------------------------------------------------------------------- + +template +void ObsErrors::save(const std::string & name) const { + for (const auto & err : err_) { + err.save(name); } } @@ -106,7 +135,7 @@ template Departures ObsErrors::inverseVariance() const { Departures_ invvar(os_); for (size_t jj = 0; jj < err_.size(); ++jj) { - invvar[jj] = err_[jj]->inverseVariance(); + invvar[jj] = err_[jj].inverseVariance(); } return invvar; } @@ -115,7 +144,7 @@ Departures ObsErrors::inverseVariance() const { template void ObsErrors::print(std::ostream & os) const { - for (size_t jj = 0; jj < err_.size(); ++jj) os << *err_[jj] << std::endl; + for (size_t jj = 0; jj < err_.size(); ++jj) os << err_[jj] << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsFilter.h b/src/oops/base/ObsFilter.h new file mode 100644 index 000000000..72e597ef4 --- /dev/null +++ b/src/oops/base/ObsFilter.h @@ -0,0 +1,184 @@ +/* + * (C) Copyright 2017-2018 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_OBSFILTER_H_ +#define OOPS_BASE_OBSFILTER_H_ + +#include +#include + +#include "oops/base/Variables.h" +#include "oops/generic/ObsFilterBase.h" +#include "oops/util/Logger.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" +#include "oops/util/Timer.h" + +namespace oops { + +template class GeoVaLs; +template class ObsDiagnostics; +template class ObsSpace; +template class ObsVector; +template class ObsDataVector; + +// ----------------------------------------------------------------------------- + +/// \brief A filter processing observations. +/// +/// Note: to see methods that need to be implemented in a generic observation filter +/// implementation, see the ObsFilterBase class in generic/ObsFilterBase.h. To see methods that +/// need to be implemented in an OBS-specific observation filter implementation, see the +/// interface::ObsFilterBase class in interface/ObsFilterBase.h. +template +class ObsFilter : public util::Printable, + private util::ObjectCounter > { + typedef GeoVaLs GeoVaLs_; + typedef ObsDiagnostics ObsDiags_; + typedef ObsFilterBase ObsFilterBase_; + typedef ObsSpace ObsSpace_; + typedef ObsVector ObsVector_; + template using ObsDataPtr_ = std::shared_ptr >; + + public: + static const std::string classname() {return "oops::ObsFilter";} + + /// \brief Create a new observation filter. + /// + /// \param obsspace + /// Space containing the observations to process. + /// \param params + /// The filter's configuration parameters. + /// \param qcflags + /// Quality control flags. They may be modified by the filter. + /// \param obserrors + /// Estimates of the standard deviations of observation errors. They may be modified by the + /// filter. + ObsFilter(const ObsSpace_ &obsspace, const ObsFilterParametersBase ¶ms, + ObsDataPtr_ qcflags, ObsDataPtr_ obserrors); + ObsFilter(const ObsFilter &) = delete; + ObsFilter(ObsFilter &&) = default; + ObsFilter& operator=(const ObsFilter &) = delete; + ObsFilter& operator=(ObsFilter &&) = default; + ~ObsFilter(); + + /// \brief Perform any observation processing steps that do not require access to GeoVaLs or + /// outputs produced by the observation operator. + void preProcess(); + + /// \brief Perform any observation processing steps that require access to GeoVaLs, but not to + /// outputs produced by the observation operator. + void priorFilter(const GeoVaLs_ &); + + /// \brief Perform any observation processing steps that require access to + /// outputs produced by the observation operator. + /// + /// \param ov + /// Model equivalents produced by the observation operator. + /// \param bv + /// Bias of departure produced by the observation operator. + /// \param dv + /// Observation diagnostics produced by the observation operator. + void postFilter(const ObsVector_ &ov, const ObsVector_ &bv, const ObsDiags_ &dv); + + /// \brief Return the list of GeoVaLs required by this filter. + Variables requiredVars() const; + + /// \brief Return the list of observation diagnostics required by this filter. + Variables requiredHdiagnostics() const; + + private: + void print(std::ostream &) const override; + + std::unique_ptr ofilt_; +}; + +// ----------------------------------------------------------------------------- + +template +ObsFilter::ObsFilter(const ObsSpace_ & os, + const ObsFilterParametersBase & parameters, + ObsDataPtr_ flags, ObsDataPtr_ obserr) +{ + Log::trace() << "ObsFilter::ObsFilter starting" << std::endl; + util::Timer timer(classname(), "ObsFilter"); + ofilt_ = FilterFactory::create(os, parameters, flags, obserr); + Log::trace() << "ObsFilter::ObsFilter done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +ObsFilter::~ObsFilter() { + Log::trace() << "ObsFilter::~ObsFilter starting" << std::endl; + util::Timer timer(classname(), "~ObsFilter"); + ofilt_.reset(); + Log::trace() << "ObsFilter::~ObsFilter done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsFilter::preProcess() { + Log::trace() << "ObsFilter::preProcess starting" << std::endl; + util::Timer timer(classname(), "preProcess"); + ofilt_->preProcess(); + Log::trace() << "ObsFilter::preProcess done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsFilter::priorFilter(const GeoVaLs_ & gv) { + Log::trace() << "ObsFilter::priorFilter starting" << std::endl; + util::Timer timer(classname(), "priorFilter"); + ofilt_->priorFilter(gv); + Log::trace() << "ObsFilter::priorFilter done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsFilter::postFilter(const ObsVector_ & ov, const ObsVector_ & bv, + const ObsDiags_ & dv) { + Log::trace() << "ObsFilter::postFilter starting" << std::endl; + util::Timer timer(classname(), "postFilter"); + ofilt_->postFilter(ov, bv, dv); + Log::trace() << "ObsFilter::postFilter done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +Variables ObsFilter::requiredVars() const { + Log::trace() << "ObsFilter::requiredVars" << std::endl; + return ofilt_->requiredVars(); +} + +// ----------------------------------------------------------------------------- + +template +Variables ObsFilter::requiredHdiagnostics() const { + Log::trace() << "ObsFilter::requiredHdiagnostics" << std::endl; + return ofilt_->requiredHdiagnostics(); +} + +// ----------------------------------------------------------------------------- + +template +void ObsFilter::print(std::ostream & os) const { + Log::trace() << "ObsFilter::print starting" << std::endl; + util::Timer timer(classname(), "print"); + os << *ofilt_; + Log::trace() << "ObsFilter::print done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSFILTER_H_ diff --git a/src/oops/base/ObsFilters.h b/src/oops/base/ObsFilters.h index 023627ee1..1ab6d0e64 100644 --- a/src/oops/base/ObsFilters.h +++ b/src/oops/base/ObsFilters.h @@ -15,13 +15,13 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/base/ObsFilterBase.h" +#include "oops/base/ObsFilter.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/IntSetParser.h" #include "oops/util/Printable.h" @@ -38,7 +38,7 @@ class ObsFilters : public util::Printable, typedef ObsDiagnostics ObsDiags_; typedef ObsSpace ObsSpace_; typedef ObsVector ObsVector_; - typedef std::shared_ptr > ObsFilterPtr_; + typedef ObsFilter ObsFilter_; typedef std::shared_ptr > ObsDataPtr_; public: @@ -49,9 +49,9 @@ class ObsFilters : public util::Printable, ObsFilters(const ObsSpace_ &, const std::vector> &, ObsDataPtr_ qcflags, ObsVector_ & obserr, const int iteration = 0); - void preProcess() const; - void priorFilter(const GeoVaLs_ &) const; - void postFilter(const ObsVector_ &, const ObsDiags_ &) const; + void preProcess(); + void priorFilter(const GeoVaLs_ &); + void postFilter(const ObsVector_ &, const ObsVector_ &, const ObsDiags_ &); Variables requiredVars() const {return geovars_;} Variables requiredHdiagnostics() const {return diagvars_;} @@ -59,7 +59,7 @@ class ObsFilters : public util::Printable, private: void print(std::ostream &) const override; - std::vector filters_; + std::vector filters_; Variables geovars_; Variables diagvars_; ObsDataPtr_ qcflags_; @@ -81,9 +81,11 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, // Prepare QC handling and statistics if any filters are present if (filtersParams.size() > 0) { - eckit::LocalConfiguration preconf; - preconf.set("filter", "QCmanager"); - filters_.push_back(FilterFactory::create(os, preconf, qcflags_, obserrtmp_)); + eckit::LocalConfiguration conf; + conf.set("filter", "QCmanager"); + ObsFilterParametersWrapper filterParams; + filterParams.validateAndDeserialize(conf); + filters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrtmp_); } // Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" @@ -96,23 +98,30 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, apply = contains(iters, iteration); } if (apply) { - ObsFilterPtr_ tmp(FilterFactory::create(os, filterParams.filterParameters, - qcflags_, obserrtmp_)); - geovars_ += tmp->requiredVars(); - diagvars_ += tmp->requiredHdiagnostics(); - filters_.push_back(tmp); + filters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrtmp_); + geovars_ += filters_.back().requiredVars(); + diagvars_ += filters_.back().requiredHdiagnostics(); } } +// Create the final filter run at the end of the pipeline + if (filtersParams.size() > 0) { + eckit::LocalConfiguration conf; + conf.set("filter", "Final Check"); + ObsFilterParametersWrapper filterParams; + filterParams.validateAndDeserialize(conf); + filters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrtmp_); + } + Log::trace() << "ObsFilters::ObsFilters done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObsFilters::preProcess() const { - for (const auto & filter : filters_) { - filter->preProcess(); +void ObsFilters::preProcess() { + for (ObsFilter_ & filter : filters_) { + filter.preProcess(); } obserrtmp_->mask(*qcflags_); obserr_ = *obserrtmp_; @@ -121,9 +130,9 @@ void ObsFilters::preProcess() const { // ----------------------------------------------------------------------------- template -void ObsFilters::priorFilter(const GeoVaLs_ & gv) const { - for (const auto & filter : filters_) { - filter->priorFilter(gv); +void ObsFilters::priorFilter(const GeoVaLs_ & gv) { + for (ObsFilter_ & filter : filters_) { + filter.priorFilter(gv); } obserrtmp_->mask(*qcflags_); obserr_ = *obserrtmp_; @@ -132,9 +141,10 @@ void ObsFilters::priorFilter(const GeoVaLs_ & gv) const { // ----------------------------------------------------------------------------- template -void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diags) const { - for (const auto & filter : filters_) { - filter->postFilter(hofx, diags); +void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsVector_ & bias, + const ObsDiags_ & diags) { + for (ObsFilter_ & filter : filters_) { + filter.postFilter(hofx, bias, diags); } obserrtmp_->mask(*qcflags_); obserr_ = *obserrtmp_; @@ -145,8 +155,8 @@ void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diag template void ObsFilters::print(std::ostream & os) const { os << "ObsFilters: " << filters_.size() << " elements:" << std::endl; - for (const auto & filter : filters_) { - os << *filter << std::endl; + for (const ObsFilter_ & filter : filters_) { + os << filter << std::endl; } } diff --git a/src/oops/base/ObsLocalizationBase.h b/src/oops/base/ObsLocalizationBase.h index 01f5711a7..57c75e5f3 100644 --- a/src/oops/base/ObsLocalizationBase.h +++ b/src/oops/base/ObsLocalizationBase.h @@ -14,8 +14,8 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/ObsVector.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsSpace.h" #include "oops/util/Printable.h" @@ -29,29 +29,26 @@ class ObsLocalizationBase : public util::Printable, private boost::noncopyable { typedef typename MODEL::GeometryIterator GeometryIterator_; typedef typename OBS::ObsVector ObsVector_; - typedef typename OBS::template ObsDataVector ObsDataVector_; public: ObsLocalizationBase() = default; virtual ~ObsLocalizationBase() = default; - /// compute obs-space localization: fill \p obsvector with observation-space - /// localization values between observations and \p point in model-space, and - /// fill \p outside with flags on whether obs is local or not (1: outside of - /// localization, 0: inside of localization, local) + /// compute obs-space localization: fill \p locfactor with observation-space + /// localization values between observations and \p point in model-space. + /// Set \p locfactor to missing value for observations that are not local. /// Method used in oops. Calls `computeLocalization` abstract method, and /// passes OBS- and MODEL-specific classes to the OBS- and MODEL-specific /// implementations of ObsLocalization. void computeLocalization(const GeometryIterator & point, - ObsDataVector & flags, ObsVector & obsvector) const { - computeLocalization(point.geometryiter(), flags.obsdatavector(), obsvector.obsvector()); + ObsVector & locfactor) const { + computeLocalization(point.geometryiter(), locfactor.obsvector()); } - /// compute obs-space localization: fill \p obsvector with observation-space - /// localization values between observations and \p point in model-space, and - /// fill \p outside with flags on whether obs is local or not (1: outside of - /// localization, 0: inside of localization, local) + /// compute obs-space localization: fill \p locfactor with observation-space + /// localization values between observations and \p point in model-space. + /// Set \p locfactor to missing value for observations that are not local. virtual void computeLocalization(const GeometryIterator_ & point, - ObsDataVector_ & flags, ObsVector_ & obsvector) const = 0; + ObsVector_ & locfactor) const = 0; }; // ============================================================================= diff --git a/src/oops/base/ObsLocalizations.h b/src/oops/base/ObsLocalizations.h index e5a4b85c1..de4e64445 100644 --- a/src/oops/base/ObsLocalizations.h +++ b/src/oops/base/ObsLocalizations.h @@ -31,17 +31,14 @@ class ObsLocalizations : public util::Printable, typedef Departures Observations_; typedef ObsLocalizationBase ObsLocalization_; typedef ObsSpaces ObsSpaces_; - typedef std::vector>> ObsDataVectors_; public: static const std::string classname() {return "oops::ObsLocalizations";} ObsLocalizations(const eckit::Configuration &, const ObsSpaces_ &); - /// schur-multiply \p obsvectors with observation-space localizations between - /// observations in \p obsspaces and \p point in model-space void computeLocalization(const GeometryIterator_ & point, - ObsDataVectors_ & local, Observations_ & obsvectors) const; + Observations_ & obsvectors) const; private: void print(std::ostream &) const; @@ -64,9 +61,9 @@ ObsLocalizations::ObsLocalizations(const eckit::Configuration & conf template void ObsLocalizations::computeLocalization(const GeometryIterator_ & point, - ObsDataVectors_ & local, Observations_ & obsvec) const { - for (size_t jj = 0; jj < obsvec.size(); ++jj) { - if (local_[jj]) local_[jj]->computeLocalization(point, *local[jj], obsvec[jj]); + Observations_ & locfactor) const { + for (size_t jj = 0; jj < local_.size(); ++jj) { + if (local_[jj]) local_[jj]->computeLocalization(point, locfactor[jj]); } } diff --git a/src/oops/base/ObsSpaceBase.cc b/src/oops/base/ObsSpaceBase.cc index cd4b1017c..52ce88eba 100644 --- a/src/oops/base/ObsSpaceBase.cc +++ b/src/oops/base/ObsSpaceBase.cc @@ -20,7 +20,7 @@ namespace oops { int ObsSpaceBase::instances_ = 0; // ----------------------------------------------------------------------------- -ObsSpaceBase::ObsSpaceBase(const eckit::Configuration & conf, const eckit::mpi::Comm & comm, +ObsSpaceBase::ObsSpaceBase(const ObsSpaceParametersBase & params, const eckit::mpi::Comm & comm, const util::DateTime & bgn, const util::DateTime & end) : winbgn_(bgn), winend_(end), instance_(++instances_) { // @@ -33,7 +33,7 @@ ObsSpaceBase::ObsSpaceBase(const eckit::Configuration & conf, const eckit::mpi:: seed_ = dt.toSeconds(); // Won't repeat if more seconds between analysis cycles than members in EDA - seed_ += conf.getInt("obs perturbations seed", 0); + seed_ += params.obsPerturbationsSeed; // 31622400 seconds max in 1 year // 12197962800 seed at this step for 2010-01-01T03:00:00Z diff --git a/src/oops/base/ObsSpaceBase.h b/src/oops/base/ObsSpaceBase.h index 3f8b18db2..09ca6db09 100644 --- a/src/oops/base/ObsSpaceBase.h +++ b/src/oops/base/ObsSpaceBase.h @@ -16,6 +16,8 @@ #include "eckit/mpi/Comm.h" #include "oops/util/DateTime.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" #include "oops/util/Printable.h" namespace eckit { @@ -24,13 +26,27 @@ namespace eckit { namespace oops { +/// \brief Base class for configuration parameters of observation spaces. +class ObsSpaceParametersBase : public oops::Parameters { + OOPS_ABSTRACT_PARAMETERS(ObsSpaceParametersBase, Parameters) + + public: + /// \brief An integer added to the seed used to initialize the random number generator + /// producing observation perturbations (if these are requested). + /// + /// The seed is a sum of this integer and terms derived from the position of the assimilation + /// window, the ensemble member index and the number of ObsSpaceBase instances created so far + /// during the lifetime of the program. + oops::Parameter obsPerturbationsSeed{"obs perturbations seed", 0, this}; +}; + // ----------------------------------------------------------------------------- /// Base class for observation spaces. class ObsSpaceBase : public util::Printable, private boost::noncopyable { public: - ObsSpaceBase(const eckit::Configuration &, const eckit::mpi::Comm &, + ObsSpaceBase(const ObsSpaceParametersBase &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &); virtual ~ObsSpaceBase() {} diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index 557998b76..c22cfd84d 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "eckit/config/LocalConfiguration.h" @@ -24,9 +25,11 @@ #include "oops/interface/ObsSpace.h" #include "oops/mpi/mpi.h" +#include "oops/util/ConfigFunctions.h" // for vectoriseAndFilter #include "oops/util/DateTime.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/Parameters.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -36,11 +39,16 @@ namespace oops { template class ObsSpaces : public util::Printable, private util::ObjectCounter > { - typedef ObsSpace ObsSpace_; + typedef ObsSpace ObsSpace_; public: + typedef typename ObsSpace_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsSpaces";} + ObsSpaces(const std::vector &, const eckit::mpi::Comm &, + const util::DateTime &, const util::DateTime &, + const eckit::mpi::Comm & time = oops::mpi::myself()); ObsSpaces(const eckit::Configuration &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm & time = oops::mpi::myself()); @@ -69,23 +77,34 @@ class ObsSpaces : public util::Printable, // ----------------------------------------------------------------------------- template -ObsSpaces::ObsSpaces(const eckit::Configuration & conf, const eckit::mpi::Comm & comm, +ObsSpaces::ObsSpaces(const std::vector & params, const eckit::mpi::Comm & comm, const util::DateTime & bgn, const util::DateTime & end, const eckit::mpi::Comm & time) : spaces_(0), wbgn_(bgn), wend_(end) { - std::vector typeconfs = conf.getSubConfigurations(); - for (std::size_t jj = 0; jj < typeconfs.size(); ++jj) { - eckit::LocalConfiguration obsconf(typeconfs[jj], "obs space"); - Log::debug() << "ObsSpaces::ObsSpaces : conf " << obsconf << std::endl; - std::shared_ptr tmp(new ObsSpace_(obsconf, comm, bgn, end, time)); - spaces_.push_back(tmp); + spaces_.reserve(params.size()); + for (const Parameters_ & param : params) { + Log::debug() << "ObsSpaces::ObsSpaces : conf " << param << std::endl; + auto tmp = std::make_shared(param, comm, bgn, end, time); + spaces_.push_back(std::move(tmp)); } ASSERT(spaces_.size() >0); } // ----------------------------------------------------------------------------- +template +ObsSpaces::ObsSpaces(const eckit::Configuration & conf, const eckit::mpi::Comm & comm, + const util::DateTime & bgn, const util::DateTime & end, + const eckit::mpi::Comm & time) + : ObsSpaces( // Split conf into subconfigurations, extract the "obs space" section from each + // of them, then validate and deserialize that section into a Parameters_ object + validateAndDeserialize(util::vectoriseAndFilter(conf, "obs space")), + comm, bgn, end, time) +{} + +// ----------------------------------------------------------------------------- + template ObsSpaces::~ObsSpaces() {} diff --git a/src/oops/base/ObsVector.h b/src/oops/base/ObsVector.h new file mode 100644 index 000000000..d7b26bae8 --- /dev/null +++ b/src/oops/base/ObsVector.h @@ -0,0 +1,136 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_OBSVECTOR_H_ +#define OOPS_BASE_OBSVECTOR_H_ + +#include +#include +#include +#include +#include + +#include "oops/interface/ObsVector.h" + +#include "oops/interface/ObsSpace.h" +#include "oops/util/gatherPrint.h" + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief ObsVector class used in oops; subclass of interface class interface::ObsVector. +/// +/// \details +/// Handles additional MPI communicator parameter \p commTime_ in the constructors +/// (for MPI distribution in time, used in oops for 4DEnVar and weak-constraint 4DVar). +/// Adds communication through time to norm and print methods. +// ----------------------------------------------------------------------------- + +template +class ObsVector : public interface::ObsVector { + typedef typename OBS::ObsVector ObsVector_; + + public: + static const std::string classname() {return "oops::ObsVector";} + + /// Creates vector from \p obsspace. If \p name is specified, reads the + /// specified \p name variable from \p obsspace. Otherwise, zero vector is created. + explicit ObsVector(const ObsSpace & obsspace, const std::string name = ""); + + /// Wraps an existing ObsVector_. + /// + /// \param obsvector The vector to wrap. + /// \param timeComm Time communicator. + ObsVector(std::unique_ptr obsvector, const eckit::mpi::Comm &timeComm); + /// Copy constructor + ObsVector(const ObsVector &); + + /// Use assignment operator (const ObsDataVector &) from the base class + using interface::ObsVector::operator=; + + /// Return the dot product between this ObsVector and \p other ObsVector + double dot_product_with(const ObsVector & other) const; + /// Return this ObsVector rms + double rms() const; + /// Number of non-masked out observations (across all MPI tasks) + unsigned int nobs() const; + + private: + void print(std::ostream &) const; + const eckit::mpi::Comm * commTime_; +}; + +// ----------------------------------------------------------------------------- + +template +ObsVector::ObsVector(const ObsSpace & os, const std::string name) + : interface::ObsVector(os, name), commTime_(&os.timeComm()) {} + +// ----------------------------------------------------------------------------- + +template +ObsVector::ObsVector(std::unique_ptr obsvector, const eckit::mpi::Comm &timeComm) + : interface::ObsVector(std::move(obsvector)), commTime_(&timeComm) {} + +// ----------------------------------------------------------------------------- + +template +ObsVector::ObsVector(const ObsVector & other): interface::ObsVector(other), + commTime_(other.commTime_) {} + +// ----------------------------------------------------------------------------- + +template +double ObsVector::dot_product_with(const ObsVector & other) const { + double zz = interface::ObsVector::dot_product_with(other); + commTime_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + return zz; +} + +// ----------------------------------------------------------------------------- + +template +double ObsVector::rms() const { + double zz = interface::ObsVector::rms(); + size_t ntot = interface::ObsVector::nobs(); + double zzz = zz * zz * static_cast(ntot); + commTime_->allReduceInPlace(zzz, eckit::mpi::Operation::SUM); + ntot = nobs(); + if (ntot > 0) { + zzz /= static_cast(ntot); + zz = std::sqrt(zzz); + } else { + zz = 0.0; + } + return zz; +} + +// ----------------------------------------------------------------------------- + +template +unsigned int ObsVector::nobs() const { + int nobs = interface::ObsVector::nobs(); + commTime_->allReduceInPlace(nobs, eckit::mpi::Operation::SUM); + return nobs; +} + +// ----------------------------------------------------------------------------- + +template +void ObsVector::print(std::ostream & os) const { + if (commTime_->size() > 1) { + gatherPrint(os, this->obsvector(), *commTime_); + } else { + os << this->obsvector(); + } +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSVECTOR_H_ diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index bdc18e2e9..c3037aec5 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -20,7 +20,7 @@ #include "oops/base/Departures.h" #include "oops/base/ObsErrors.h" #include "oops/base/ObsSpaces.h" -#include "oops/interface/ObsVector.h" +#include "oops/base/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h index 35832ffeb..2871275e0 100644 --- a/src/oops/base/Observer.h +++ b/src/oops/base/Observer.h @@ -14,11 +14,12 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" #include "oops/base/GetValuePost.h" -#include "oops/base/ObsErrorBase.h" +#include "oops/base/ObsError.h" #include "oops/base/ObsFilters.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/Locations.h" #include "oops/interface/ObsAuxControl.h" @@ -26,7 +27,6 @@ #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" @@ -38,9 +38,15 @@ template class ObserverParameters : public Parameters { OOPS_CONCRETE_PARAMETERS(ObserverParameters, Parameters) + typedef typename OBS::ObsOperator::Parameters_ ObsOperatorParameters_; + public: - oops::RequiredParameter obsOperator{"obs operator", this}; + oops::RequiredParameter obsOperator{"obs operator", this}; oops::Parameter>> obsFilters{"obs filters", {}, this}; + oops::Parameter getValues{ + "get values", eckit::LocalConfiguration(), this}; + oops::Parameter linearGetValues{ + "linear get values", eckit::LocalConfiguration(), this}; }; // ----------------------------------------------------------------------------- @@ -55,7 +61,8 @@ class Observer { typedef ObsAuxControl ObsAuxCtrl_; typedef ObsDataVector ObsDataInt_; typedef ObsDiagnostics ObsDiags_; - typedef ObsErrorBase ObsError_; + typedef ObsError ObsError_; + typedef ObserverParameters Parameters_; typedef ObsFilters ObsFilters_; typedef ObsOperator ObsOperator_; typedef ObsSpace ObsSpace_; @@ -63,42 +70,41 @@ class Observer { public: /// \brief Initializes ObsOperators, Locations, and QC data - Observer(const ObsSpace_ &, const eckit::Configuration &); + Observer(const ObsSpace_ &, const Parameters_ &); /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, - ObsError_ &, const int iter); + ObsError_ &, const eckit::Configuration &); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(ObsVector_ &); private: - eckit::LocalConfiguration obsconfig_; + Parameters_ parameters_; const ObsSpace_ & obspace_; // ObsSpace used in H(x) std::unique_ptr obsop_; // Obs operator std::unique_ptr locations_; // locations - const ObsAuxCtrl_ * ybias_; // Obs bias + const ObsAuxCtrl_ * biascoeff_; // bias coefficients ObsError_ * Rmat_; // Obs error covariance std::unique_ptr filters_; // QC filters + std::unique_ptr obserr_; // Obs error std dev std::shared_ptr getvals_; // Postproc passed to the model during integration. std::shared_ptr qcflags_; // QC flags (should not be a pointer) - int iterout_; // Outer iteration bool initialized_; + std::unique_ptr iterconf_; }; // ----------------------------------------------------------------------------- template -Observer::Observer(const ObsSpace_ & obspace, const eckit::Configuration & config) - : obsconfig_(config), obspace_(obspace), obsop_(), locations_(), - ybias_(nullptr), filters_(), qcflags_(), iterout_(-1), initialized_(false) +Observer::Observer(const ObsSpace_ & obspace, const Parameters_ & params) + : parameters_(params), obspace_(obspace), obsop_(), locations_(), + biascoeff_(nullptr), filters_(), qcflags_(), initialized_(false) { Log::trace() << "Observer::Observer start" << std::endl; - ObserverParameters observerParams; - observerParams.deserialize(config); /// Set up observation operators - obsop_.reset(new ObsOperator_(obspace_, observerParams.obsOperator)); + obsop_.reset(new ObsOperator_(obspace_, parameters_.obsOperator)); qcflags_.reset(new ObsDataInt_(obspace_, obspace_.obsvariables())); Log::trace() << "Observer::Observer done" << std::endl; @@ -108,20 +114,19 @@ Observer::Observer(const ObsSpace_ & obspace, const eckit::Configura template std::shared_ptr> -Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, - ObsError_ & R, const int iter) { +Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & biascoeff, + ObsError_ & R, const eckit::Configuration & conf) { Log::trace() << "Observer::initialize start" << std::endl; // Save information for finalize - iterout_ = iter; - ybias_ = &ybias; + iterconf_.reset(new eckit::LocalConfiguration(conf)); + biascoeff_ = &biascoeff; Rmat_ = &R; + obserr_.reset(new ObsVector_(Rmat_->obserrors())); // Set up QC filters and run preprocess - int iterfilt = std::max(iter, 0); - ObserverParameters observerParams; - observerParams.deserialize(obsconfig_); - filters_.reset(new ObsFilters_(obspace_, observerParams.obsFilters, - qcflags_, Rmat_->obserrors(), iterfilt)); + const int iterfilt = iterconf_->getInt("iteration", 0); + filters_.reset(new ObsFilters_(obspace_, parameters_.obsFilters, + qcflags_, *obserr_, iterfilt)); filters_->preProcess(); locations_.reset(new Locations_(obsop_->locations())); @@ -129,13 +134,11 @@ Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybi // Set up variables that will be requested from the model Variables geovars; geovars += obsop_->requiredVars(); - geovars += ybias_->requiredVars(); + geovars += biascoeff_->requiredVars(); geovars += filters_->requiredVars(); - eckit::LocalConfiguration gvconf = obsconfig_.getSubConfiguration("get values"); - // Set up GetValues - getvals_.reset(new GetValPost_(gvconf, geom, obspace_.windowStart(), + getvals_.reset(new GetValPost_(parameters_.getValues, geom, obspace_.windowStart(), obspace_.windowEnd(), *locations_, geovars)); initialized_ = true; @@ -159,27 +162,42 @@ void Observer::finalize(ObsVector_ & yobsim) { /// Setup diagnostics Variables vars; vars += filters_->requiredHdiagnostics(); - vars += ybias_->requiredHdiagnostics(); + vars += biascoeff_->requiredHdiagnostics(); ObsDiags_ ydiags(obspace_, *locations_, vars); + // Setup bias vector + ObsVector_ ybias(obspace_); + ybias.zero(); + /// Compute H(x) - obsop_->simulateObs(*geovals, yobsim, *ybias_, ydiags); + obsop_->simulateObs(*geovals, yobsim, *biascoeff_, ybias, ydiags); /// Call posterior filters - filters_->postFilter(yobsim, ydiags); + filters_->postFilter(yobsim, ybias, ydiags); // Update R with obs errors that filters might have updated - Rmat_->update(); + Rmat_->update(*obserr_); // Save current obs, obs error estimates and QC flags (for diagnostics use only) std::string siter = ""; - if (iterout_ >= 0) siter = std::to_string(iterout_); - const std::string qcname = "EffectiveQC" + siter; - qcflags_->save(qcname); - const std::string obsname = "hofx" + siter; - yobsim.save(obsname); - const std::string errname = "EffectiveError" + siter; - Rmat_->save(errname); + if (iterconf_->has("iteration")) siter = iterconf_->getString("iteration"); + + if (iterconf_->getBool("save qc", true)) { + const std::string qcname = "EffectiveQC" + siter; + qcflags_->save(qcname); + } + if (iterconf_->getBool("save hofx", true)) { + const std::string obsname = "hofx" + siter; + yobsim.save(obsname); + } + if (iterconf_->getBool("save obs errors", true)) { + const std::string errname = "EffectiveError" + siter; + Rmat_->save(errname); + } + if (iterconf_->getBool("save obs bias", true)) { + const std::string biasname = "ObsBias" + siter; + ybias.save(biasname); + } Log::info() << "Observer::finalize QC = " << *qcflags_ << std::endl; diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index dc0b8c22c..7dcebf1d5 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -15,8 +15,9 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" #include "oops/base/GetValueTLAD.h" -#include "oops/interface/Geometry.h" +#include "oops/base/ObsVector.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/LinearObsOperator.h" #include "oops/interface/Locations.h" @@ -24,7 +25,6 @@ #include "oops/interface/ObsAuxIncrement.h" #include "oops/interface/ObsOperator.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/DateTime.h" namespace oops { @@ -62,6 +62,8 @@ class ObserverTLAD { const ObsSpace_ & obspace_; // ObsSpace used in H(x) LinearObsOperator_ hoptlad_; // Linear obs operator std::shared_ptr getvals_; // Postproc passed to the model during integration + std::vector linvars_sizes_; // Sizes of variables requested from model for + // TL/AD (e.g. number of vertical levels) std::unique_ptr locations_; // locations util::DateTime winbgn_; // Begining of assimilation window util::DateTime winend_; // End of assimilation window @@ -73,9 +75,11 @@ class ObserverTLAD { template ObserverTLAD::ObserverTLAD(const ObsSpace_ & obsdb, const eckit::Configuration & conf) : obsconfig_(conf), obspace_(obsdb), - hoptlad_(obspace_, conf.has("linear obs operator") ? + hoptlad_(obspace_, + validateAndDeserialize( + conf.has("linear obs operator") ? eckit::LocalConfiguration(conf, "linear obs operator") : - eckit::LocalConfiguration(conf, "obs operator")), + eckit::LocalConfiguration(conf, "obs operator"))), getvals_(), locations_(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), ybias_(nullptr), init_(false) { @@ -89,9 +93,11 @@ ObserverTLAD::initializeTraj(const Geometry_ & geom, const ObsAuxCtr ybias_ = &ybias; // hop is only needed to get locations and requiredVars - ObsOperator_ hop(obspace_, eckit::LocalConfiguration(obsconfig_, "obs operator")); + ObsOperator_ hop(obspace_, + validateAndDeserialize( + eckit::LocalConfiguration(obsconfig_, "obs operator"))); locations_.reset(new Locations_(hop.locations())); - + linvars_sizes_ = geom.variableSizes(hoptlad_.requiredVars()); eckit::LocalConfiguration gvconf(obsconfig_.has("linear get values") ? eckit::LocalConfiguration(obsconfig_, "linear get values") : (obsconfig_.has("get values") ? @@ -151,7 +157,8 @@ ObserverTLAD::initializeAD(const ObsVector_ & ydepad, ObsAuxIncr_ & Log::trace() << "ObserverTLAD::initializeAD start" << std::endl; // Compute adjoint of H(x) - std::unique_ptr geovals(new GeoVaLs_(*locations_, hoptlad_.requiredVars())); + std::unique_ptr geovals(new GeoVaLs_(*locations_, hoptlad_.requiredVars(), + linvars_sizes_)); hoptlad_.simulateObsAD(*geovals, ydepad, ybiasad); // GetValues get GeoVaLs and takes ownership diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index 9f418470a..aa4dd34b7 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -14,16 +14,16 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" #include "oops/base/GetValuePosts.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" #include "oops/base/Observer.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/ObsVector.h" #include "oops/base/PostProcessor.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/ObsVector.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/util/Logger.h" namespace oops { @@ -40,23 +40,30 @@ class Observers { typedef ObsErrors ObsErrors_; typedef Observations Observations_; typedef Observer Observer_; + typedef ObserverParameters ObserverParameters_; typedef ObsSpaces ObsSpaces_; typedef ObsVector ObsVector_; typedef State State_; typedef PostProcessor PostProc_; + template using ObsData_ = ObsDataVector; + template using ObsDataVec_ = std::vector>>; public: /// \brief Initializes ObsOperators, Locations, and QC data + Observers(const ObsSpaces_ &, const std::vector &); Observers(const ObsSpaces_ &, const eckit::Configuration &); /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations void initialize(const Geometry_ &, const ObsAuxCtrls_ &, ObsErrors_ &, - PostProc_ &, const int iter = -1); + PostProc_ &, const eckit::Configuration & = eckit::LocalConfiguration()); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(Observations_ &); + private: + static std::vector convertToParameters(const eckit::Configuration &config); + private: std::vector> observers_; }; @@ -64,14 +71,15 @@ class Observers { // ----------------------------------------------------------------------------- template -Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Configuration & config) +Observers::Observers(const ObsSpaces_ & obspaces, + const std::vector ¶ms) : observers_() { Log::trace() << "Observers::Observers start" << std::endl; - std::vector obsconfs = config.getSubConfigurations(); + ASSERT(obspaces.size() == params.size()); for (size_t jj = 0; jj < obspaces.size(); ++jj) { - observers_.emplace_back(new Observer_(obspaces[jj], obsconfs[jj])); + observers_.emplace_back(new Observer_(obspaces[jj], params[jj])); } Log::trace() << "Observers::Observers done" << std::endl; @@ -79,14 +87,22 @@ Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Confi // ----------------------------------------------------------------------------- +template +Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Configuration & config) + : Observers(obspaces, convertToParameters(config)) +{} + +// ----------------------------------------------------------------------------- + template void Observers::initialize(const Geometry_ & geom, const ObsAuxCtrls_ & obsaux, - ObsErrors_ & Rmat, PostProc_ & pp, const int iter) { + ObsErrors_ & Rmat, PostProc_ & pp, + const eckit::Configuration & conf) { Log::trace() << "Observers::initialize start" << std::endl; std::shared_ptr getvals(new GetValuePosts_()); for (size_t jj = 0; jj < observers_.size(); ++jj) { - getvals->append(observers_[jj]->initialize(geom, obsaux[jj], Rmat[jj], iter)); + getvals->append(observers_[jj]->initialize(geom, obsaux[jj], Rmat[jj], conf)); } pp.enrollProcessor(getvals); @@ -108,6 +124,44 @@ void Observers::finalize(Observations_ & yobs) { // ----------------------------------------------------------------------------- +template +std::vector> Observers::convertToParameters( + const eckit::Configuration &config) { + oops::Log::trace() << "Observers::convertToParameters start" << std::endl; + + std::vector subconfigs = config.getSubConfigurations(); + std::vector> parameters(subconfigs.size()); + for (size_t i = 0; i < subconfigs.size(); ++i) { + const eckit::LocalConfiguration &subconfig = subconfigs[i]; + + // 'subconfig' will, in general, contain options irrelevant to the observer (e.g. 'obs space'). + // So we need to extract the relevant parts into a new Configuration object, 'observerConfig', + // before validation and deserialization. Otherwise validation might fail. + + eckit::LocalConfiguration observerConfig; + + // Required keys + observerConfig.set("obs operator", eckit::LocalConfiguration(subconfig, "obs operator")); + + // Optional keys + std::vector filterConfigs; + if (subconfig.get("obs filters", filterConfigs)) + observerConfig.set("obs filters", filterConfigs); + eckit::LocalConfiguration getValuesConfig; + if (subconfig.get("get values", getValuesConfig)) + observerConfig.set("get values", getValuesConfig); + eckit::LocalConfiguration linearGetValuesConfig; + if (subconfig.get("linear get values", linearGetValuesConfig)) + observerConfig.set("linear get values", linearGetValuesConfig); + + parameters[i].validateAndDeserialize(observerConfig); + } + + oops::Log::trace() << "Observers::convertToParameters start" << std::endl; + + return parameters; +} + } // namespace oops #endif // OOPS_BASE_OBSERVERS_H_ diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index bb2731b9c..5e3d2c32e 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -19,13 +19,13 @@ #include "eckit/config/Configuration.h" #include "oops/base/Departures.h" #include "oops/base/GeneralizedDepartures.h" +#include "oops/base/Geometry.h" #include "oops/base/GetValueTLADs.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsAuxIncrements.h" #include "oops/base/ObserverTLAD.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessorTLAD.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/PostBaseTLAD.h b/src/oops/base/PostBaseTLAD.h index e4cc17c53..43bb9e6e8 100644 --- a/src/oops/base/PostBaseTLAD.h +++ b/src/oops/base/PostBaseTLAD.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -17,9 +17,9 @@ #include "eckit/config/Configuration.h" #include "oops/base/GeneralizedDepartures.h" +#include "oops/base/Increment.h" #include "oops/base/PostTimer.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/PostProcessorTLAD.h b/src/oops/base/PostProcessorTLAD.h index 8d27b2946..6cb5c9980 100644 --- a/src/oops/base/PostProcessorTLAD.h +++ b/src/oops/base/PostProcessorTLAD.h @@ -16,9 +16,9 @@ #include #include "oops/base/GeneralizedDepartures.h" +#include "oops/base/Increment.h" #include "oops/base/PostBaseTLAD.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" namespace oops { diff --git a/src/oops/base/State.h b/src/oops/base/State.h new file mode 100644 index 000000000..fef37878e --- /dev/null +++ b/src/oops/base/State.h @@ -0,0 +1,100 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_STATE_H_ +#define OOPS_BASE_STATE_H_ + +#include +#include +#include + +#include "oops/interface/State.h" + + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief State class used in oops; subclass of interface class interface::State. +/// +/// \details +/// Handles additional MPI communicator parameter \p commTime_ in the constructors +/// (for MPI distribution in time, used in oops for 4DEnVar and weak-constraint 4DVar). +/// Adds communication through time to the following Increment methods: +/// - norm +/// - print + +// ----------------------------------------------------------------------------- +template +class State : public interface::State { + typedef typename MODEL::State State_; + typedef Geometry Geometry_; + + public: + /// Constructor for specified \p resol, with \p vars, valid at \p time + State(const Geometry_ & resol, const Variables & vars, const util::DateTime & time); + /// Constructor for specified \p resol and files read from \p conf + State(const Geometry_ & resol, const eckit::Configuration & conf); + /// Copies \p other State, changing its resolution to \p geometry + State(const Geometry_ & resol, const State & other); + + /// Norm (used in tests) + double norm() const; + + private: + const eckit::mpi::Comm * commTime_; /// pointer to the MPI communicator in time + void print(std::ostream &) const override; +}; + +// ============================================================================= + +template +State::State(const Geometry_ & resol, const Variables & vars, + const util::DateTime & time) : + interface::State(resol, vars, time), commTime_(&resol.timeComm()) +{} + +// ----------------------------------------------------------------------------- + +template +State::State(const Geometry_ & resol, const eckit::Configuration & conf) : + interface::State(resol, conf), commTime_(&resol.timeComm()) +{} + +// ----------------------------------------------------------------------------- + +template +State::State(const Geometry_ & resol, const State & other) : + interface::State(resol, other), commTime_(&resol.timeComm()) +{} + +// ----------------------------------------------------------------------------- + +template +double State::norm() const { + double zz = interface::State::norm(); + zz *= zz; + commTime_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + zz = sqrt(zz); + return zz; +} + +// ----------------------------------------------------------------------------- + +template +void State::print(std::ostream & os) const { + if (commTime_->size() > 1) { + gatherPrint(os, this->state(), *commTime_); + } else { + os << this->state(); + } +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_STATE_H_ diff --git a/src/oops/assimilation/State4D.h b/src/oops/base/State4D.h similarity index 96% rename from src/oops/assimilation/State4D.h rename to src/oops/base/State4D.h index 9c0035fa6..bddbae425 100644 --- a/src/oops/assimilation/State4D.h +++ b/src/oops/base/State4D.h @@ -9,8 +9,8 @@ * does it submit to any jurisdiction. */ -#ifndef OOPS_ASSIMILATION_STATE4D_H_ -#define OOPS_ASSIMILATION_STATE4D_H_ +#ifndef OOPS_BASE_STATE4D_H_ +#define OOPS_BASE_STATE4D_H_ #include #include @@ -18,8 +18,8 @@ #include "eckit/config/LocalConfiguration.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" @@ -149,4 +149,4 @@ void State4D::print(std::ostream & outs) const { } // namespace oops -#endif // OOPS_ASSIMILATION_STATE4D_H_ +#endif // OOPS_BASE_STATE4D_H_ diff --git a/src/oops/base/StateEnsemble.h b/src/oops/base/StateEnsemble.h index 6132173f6..17e787d21 100644 --- a/src/oops/base/StateEnsemble.h +++ b/src/oops/base/StateEnsemble.h @@ -13,9 +13,9 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/base/Accumulator.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" namespace oops { diff --git a/src/oops/base/StateEnsemble4D.h b/src/oops/base/StateEnsemble4D.h index 93f969573..c09e5d468 100644 --- a/src/oops/base/StateEnsemble4D.h +++ b/src/oops/base/StateEnsemble4D.h @@ -12,10 +12,10 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/State4D.h" #include "oops/base/Accumulator.h" +#include "oops/base/Geometry.h" +#include "oops/base/State4D.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/util/Logger.h" namespace oops { diff --git a/src/oops/base/TrajectorySaver.h b/src/oops/base/TrajectorySaver.h index 46c55db06..169c5ae29 100644 --- a/src/oops/base/TrajectorySaver.h +++ b/src/oops/base/TrajectorySaver.h @@ -14,11 +14,11 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/LinearModel.h" #include "oops/base/PostBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/LinearModel.h" +#include "oops/base/State.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/VariableChangeBase.h b/src/oops/base/VariableChangeBase.h index b4487e495..0e96aea5c 100644 --- a/src/oops/base/VariableChangeBase.h +++ b/src/oops/base/VariableChangeBase.h @@ -16,9 +16,9 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/VariableChangeParametersBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/util/AssociativeContainers.h" #include "oops/util/parameters/ConfigurationParameter.h" #include "oops/util/parameters/HasParameters_.h" diff --git a/src/oops/base/Variables.cc b/src/oops/base/Variables.cc index 927ff6f99..b11a28e39 100644 --- a/src/oops/base/Variables.cc +++ b/src/oops/base/Variables.cc @@ -35,8 +35,7 @@ Variables::Variables(const eckit::Configuration & conf, const std::string & name : convention_(""), vars_(0), channels_(0) { Log::trace() << "Variables::Variables start " << conf << std::endl; std::vector vars; - conf.get(name, vars); - if (vars.size() == 0) { + if (!conf.get(name, vars)) { Log::error() << name << " not found in " << conf << std::endl; throw eckit::BadParameter("Undefined variable: '" + name + "'"); } diff --git a/src/oops/base/WeightedDiff.h b/src/oops/base/WeightedDiff.h index 1ce07836e..8657c5c22 100644 --- a/src/oops/base/WeightedDiff.h +++ b/src/oops/base/WeightedDiff.h @@ -16,10 +16,10 @@ #include "oops/base/Accumulator.h" #include "oops/base/DolphChebyshev.h" +#include "oops/base/Geometry.h" #include "oops/base/PostBase.h" #include "oops/base/Variables.h" #include "oops/base/WeightingFct.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" diff --git a/src/oops/base/WeightedDiffTLAD.h b/src/oops/base/WeightedDiffTLAD.h index 44c068734..26048ec02 100644 --- a/src/oops/base/WeightedDiffTLAD.h +++ b/src/oops/base/WeightedDiffTLAD.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -18,13 +18,13 @@ #include "oops/base/Accumulator.h" #include "oops/base/DolphChebyshev.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/PostBaseTLAD.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/base/WeightedDiff.h" #include "oops/base/WeightingFct.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/WeightedMean.h b/src/oops/base/WeightedMean.h index 4d73a4019..256b23fa2 100644 --- a/src/oops/base/WeightedMean.h +++ b/src/oops/base/WeightedMean.h @@ -19,10 +19,10 @@ #include "oops/base/Accumulator.h" #include "oops/base/DolphChebyshev.h" +#include "oops/base/Geometry.h" #include "oops/base/PostBase.h" #include "oops/base/Variables.h" #include "oops/base/WeightingFct.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" diff --git a/src/oops/base/instantiateObsFilterFactory.h b/src/oops/base/instantiateObsFilterFactory.h index fc8219ead..199a6b7d8 100644 --- a/src/oops/base/instantiateObsFilterFactory.h +++ b/src/oops/base/instantiateObsFilterFactory.h @@ -8,13 +8,13 @@ #ifndef OOPS_BASE_INSTANTIATEOBSFILTERFACTORY_H_ #define OOPS_BASE_INSTANTIATEOBSFILTERFACTORY_H_ -#include "oops/base/GeoVaLsWriter.h" -#include "oops/base/ObsFilterBase.h" +#include "oops/generic/GeoVaLsWriter.h" +#include "oops/generic/ObsFilterBase.h" namespace oops { template void instantiateObsFilterFactory() { - static FilterMaker > makerGVWriter_("GOMsaver"); + static FilterMaker> makerGVWriter_("GOMsaver"); } } // namespace oops diff --git a/src/oops/base/GeoVaLsWriter.h b/src/oops/generic/GeoVaLsWriter.h similarity index 84% rename from src/oops/base/GeoVaLsWriter.h rename to src/oops/generic/GeoVaLsWriter.h index aa59778f1..afa78703d 100644 --- a/src/oops/base/GeoVaLsWriter.h +++ b/src/oops/generic/GeoVaLsWriter.h @@ -5,19 +5,19 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_BASE_GEOVALSWRITER_H_ -#define OOPS_BASE_GEOVALSWRITER_H_ +#ifndef OOPS_GENERIC_GEOVALSWRITER_H_ +#define OOPS_GENERIC_GEOVALSWRITER_H_ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/base/ObsFilterBase.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" +#include "oops/generic/ObsFilterBase.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" @@ -38,15 +38,15 @@ class GeoVaLsWriter : public ObsFilterBase { ObsDataPtr_, ObsDataPtr_): conf_(conf), novars_() {} ~GeoVaLsWriter() {} - void preProcess() const override {} + void preProcess() override {} - void priorFilter(const GeoVaLs_ & gv) const override { + void priorFilter(const GeoVaLs_ & gv) override { const double zz = sqrt(dot_product(gv, gv)); Log::info() << "GeoVaLsWriter norm = " << zz << std::endl; gv.write(conf_); } - void postFilter(const ObsVector_ &, const ObsDiags_ &) const override {} + void postFilter(const ObsVector_ &, const ObsVector_ &, const ObsDiags_ &) override {} Variables requiredVars() const override {return novars_;}; Variables requiredHdiagnostics() const override {return novars_;}; @@ -69,4 +69,4 @@ void GeoVaLsWriter::print(std::ostream & os) const { } // namespace oops -#endif // OOPS_BASE_GEOVALSWRITER_H_ +#endif // OOPS_GENERIC_GEOVALSWRITER_H_ diff --git a/src/oops/generic/IdLinearVariableChange.h b/src/oops/generic/IdLinearVariableChange.h index e615e9ca3..1ecd80f0b 100644 --- a/src/oops/generic/IdLinearVariableChange.h +++ b/src/oops/generic/IdLinearVariableChange.h @@ -11,10 +11,10 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/State.h" #include "oops/util/Printable.h" // Forward declarations diff --git a/src/oops/generic/IdVariableChange.h b/src/oops/generic/IdVariableChange.h index 8e51e720e..d7ec2b79b 100644 --- a/src/oops/generic/IdVariableChange.h +++ b/src/oops/generic/IdVariableChange.h @@ -11,9 +11,9 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/VariableChangeBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/util/Printable.h" // Forward declarations diff --git a/src/oops/generic/IdentityLinearModel.h b/src/oops/generic/IdentityLinearModel.h new file mode 100644 index 000000000..d7c2113b9 --- /dev/null +++ b/src/oops/generic/IdentityLinearModel.h @@ -0,0 +1,149 @@ +/* + * (C) Copyright 2020-2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_GENERIC_IDENTITYLINEARMODEL_H_ +#define OOPS_GENERIC_IDENTITYLINEARMODEL_H_ + +#include + +#include "oops/base/Geometry.h" +#include "oops/base/ParameterTraitsVariables.h" +#include "oops/base/State.h" +#include "oops/base/Variables.h" +#include "oops/generic/LinearModelBase.h" +#include "oops/interface/Increment.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/interface/ModelAuxIncrement.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace oops { + +class IdentityLinearModelParameters : public LinearModelParametersBase { + OOPS_CONCRETE_PARAMETERS(IdentityLinearModelParameters, LinearModelParametersBase) + + public: + oops::RequiredParameter tstep{"tstep", this}; + oops::RequiredParameter vars{"increment variables", this}; + // This does not really belong here but adding as a quick fix + oops::OptionalParameter variableChange{"variable change", this}; +}; + +/// Generic implementation of identity linear model +template +class IdentityLinearModel : public LinearModelBase { + typedef Geometry Geometry_; + typedef Increment Increment_; + typedef ModelAuxControl ModelAuxCtl_; + typedef ModelAuxIncrement ModelAuxInc_; + typedef State State_; + + public: + typedef IdentityLinearModelParameters Parameters_; + + static const std::string classname() {return "oops::IdentityLinearModel";} + + IdentityLinearModel(const Geometry_ &, const IdentityLinearModelParameters &); + +/// initialize tangent linear forecast + void initializeTL(Increment_ &) const override; +/// one tangent linear forecast step + void stepTL(Increment_ &, const ModelAuxInc_ &) const override; +/// finalize tangent linear forecast + void finalizeTL(Increment_ &) const override; + +/// initialize tangent linear forecast + void initializeAD(Increment_ &) const override; +/// one tangent linear forecast step + void stepAD(Increment_ &, ModelAuxInc_ &) const override; +/// finalize tangent linear forecast + void finalizeAD(Increment_ &) const override; + +/// set trajectory + void setTrajectory(const State_ &, State_ &, const ModelAuxCtl_ &) override; + +/// linear model time step + const util::Duration & timeResolution() const override {return params_.tstep;} +/// linear model variables + const oops::Variables & variables() const override {return params_.vars;} + + private: + void print(std::ostream &) const override {} + const IdentityLinearModelParameters params_; +}; + +// ----------------------------------------------------------------------------- + +template +IdentityLinearModel::IdentityLinearModel(const Geometry_ & resol, + const IdentityLinearModelParameters & params) + : params_(params) { + Log::trace() << "IdentityLinearModel::IdentityLinearModel done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::initializeTL(Increment_ & dx) const { + Log::trace() << "IdentityLinearModel::initializeTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::stepTL(Increment_ & dx, const ModelAuxInc_ & merr) const { + Log::trace() << "IdentityLinearModel:stepTL Starting " << std::endl; + dx.updateTime(params_.tstep); + Log::trace() << "IdentityLinearModel::stepTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::finalizeTL(Increment_ & dx) const { + Log::trace() << "IdentityLinearModel::finalizeTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::initializeAD(Increment_ & dx) const { + Log::trace() << "IdentityLinearModel::initializeAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::stepAD(Increment_ & dx, ModelAuxInc_ & merr) const { + Log::trace() << "IdentityLinearModel:stepAD Starting " << std::endl; + const util::Duration tstep = params_.tstep; + dx.updateTime(-tstep); + Log::trace() << "IdentityLinearModel::stepAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::finalizeAD(Increment_ & dx) const { + Log::trace() << "IdentityLinearModel::finalizeAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void IdentityLinearModel::setTrajectory(const State_ & x, State_ & xlr, + const ModelAuxCtl_ & maux) { + Log::trace() << "IdentityLinearModel::finalizeAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_GENERIC_IDENTITYLINEARMODEL_H_ diff --git a/src/oops/generic/IdentityModel.h b/src/oops/generic/IdentityModel.h index 8d3ccfe94..8e809385b 100644 --- a/src/oops/generic/IdentityModel.h +++ b/src/oops/generic/IdentityModel.h @@ -10,12 +10,12 @@ #include -#include "oops/base/ModelBase.h" +#include "oops/base/Geometry.h" #include "oops/base/ParameterTraitsVariables.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" +#include "oops/generic/ModelBase.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameters.h" @@ -33,7 +33,7 @@ class IdentityModelParameters : public ModelParametersBase { /// Generic implementation of identity model template -class IdentityModel : public GenericModelBase { +class IdentityModel : public ModelBase { typedef Geometry Geometry_; typedef ModelAuxControl ModelAux_; typedef State State_; diff --git a/src/oops/generic/LinearModelBase.h b/src/oops/generic/LinearModelBase.h new file mode 100644 index 000000000..fe2c1dd73 --- /dev/null +++ b/src/oops/generic/LinearModelBase.h @@ -0,0 +1,273 @@ +/* + * (C) Copyright 2018-2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_GENERIC_LINEARMODELBASE_H_ +#define OOPS_GENERIC_LINEARMODELBASE_H_ + +#include +#include +#include +#include + +#include +#include + +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/interface/ModelAuxIncrement.h" +#include "oops/util/AssociativeContainers.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/HasParameters_.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/ParametersOrConfiguration.h" +#include "oops/util/parameters/RequiredPolymorphicParameter.h" +#include "oops/util/Printable.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +// ----------------------------------------------------------------------------- + +/// \brief Base class for generic implementations of the linearized forecasting models. +/// Use this class as a base class for generic implementations, +/// and interface::LinearModelBase as a base class for MODEL-specific implementations. +/// +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of LinearModelParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// LinearModelBase(const Geometry_ &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of LinearModelParametersBase +/// holding the settings of the linear model in question. The implementation of the LinearModelBase +/// interface should then typedef `Parameters_` to the name of that subclass and provide a +/// constructor with the following signature: +/// +/// LinearModelBase(const Geometry_ &, const Parameters_ &); +/// +template +class LinearModelBase : public util::Printable, + private boost::noncopyable { + typedef Increment Increment_; + typedef ModelAuxControl ModelAuxCtl_; + typedef ModelAuxIncrement ModelAuxInc_; + typedef State State_; + + public: + static const std::string classname() {return "oops::LinearModelBase";} + + LinearModelBase() = default; + virtual ~LinearModelBase() = default; + + /// \brief Tangent linear initialization, called before every run + virtual void initializeTL(Increment_ &) const = 0; + /// \brief Tangent linear "step", called during run; updates increment to the next time + virtual void stepTL(Increment_ &, const ModelAuxInc_ &) const = 0; + /// \brief Tangent linear finalization; called after each run + virtual void finalizeTL(Increment_ &) const = 0; + + /// \brief Tangent linear initialization, called before every run + virtual void initializeAD(Increment_ &) const = 0; + /// \brief Tangent linear "step", called during run; updates increment to the next time + virtual void stepAD(Increment_ &, ModelAuxInc_ &) const = 0; + /// \brief Tangent linear finalization; called after each run + virtual void finalizeAD(Increment_ &) const = 0; + + /// \brief Set the trajectory for the linear model, called after each step of the forecast + virtual void setTrajectory(const State_ &, State_ &, const ModelAuxCtl_ &) = 0; + + /// \brief Time step for running LinearModel's forecast in oops (frequency with which the + /// increment will be updated) + virtual const util::Duration & timeResolution() const = 0; + /// \brief LinearModel variables (only used in 4DVar) + virtual const oops::Variables & variables() const = 0; + + private: + /// \brief Print; used for logging + virtual void print(std::ostream &) const = 0; +}; + +// ============================================================================= + +template +class LinearModelFactory; + +// ----------------------------------------------------------------------------- + +/// \brief Base class for classes storing linear model-specific parameters. +class LinearModelParametersBase : public Parameters { + OOPS_ABSTRACT_PARAMETERS(LinearModelParametersBase, Parameters) + public: + /// \brief LinearModel name. + /// + /// \note This parameter is marked as optional because it is only required in certain + /// circumstances (e.g. when linear model parameters are deserialized into a + /// LinearModelParametersWrapper and used by LinearModelFactory to instantiate a linear model + /// whose type is determined at runtime), but not others (e.g. in tests written with a particular + /// linear model in mind). LinearModelParametersWrapper will throw an exception if this parameter + /// is not provided. + OptionalParameter name{"name", this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of LinearModelParametersBase storing the values of all options in a +/// single Configuration object. +/// +/// This object can be accessed by calling the value() method of the \p config member variable. +/// +/// The ConfigurationParameter class does not perform any parameter validation; linear models using +/// GenericLinearModelParameters should therefore ideally be refactored, replacing this class with a +/// dedicated subclass of LinearModelParametersBase storing each parameter in a separate +/// (Optional/Required)Parameter object. +class GenericLinearModelParameters : public LinearModelParametersBase { + OOPS_CONCRETE_PARAMETERS(GenericLinearModelParameters, LinearModelParametersBase) + public: + ConfigurationParameter config{this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// LinearModelParametersBase. +template +class LinearModelParametersWrapper : public Parameters { + OOPS_CONCRETE_PARAMETERS(LinearModelParametersWrapper, Parameters) + public: + /// After deserializtion, holds an instance of a subclass of LinearModelParametersBase controlling + /// the behavior of a linear model. The type of the subclass is determined by the value of the + /// "name" key in the Configuration object from which this object is deserialized. + RequiredPolymorphicParameter> + linearModelParameters{"name", this}; +}; + +// ============================================================================= + +/// LinearModel factory +template +class LinearModelFactory { + typedef Geometry Geometry_; + + public: + /// \brief Create and return a new linear model. + /// + /// The linear model's type is determined by the \c name attribute of \p parameters. + /// \p parameters must be an instance of the subclass of LinearModelParametersBase + /// associated with that linear model type, otherwise an exception will be thrown. + static LinearModelBase * create(const Geometry_ &, + const LinearModelParametersBase & parameters); + + /// \brief Create and return an instance of the subclass of LinearModelParametersBase + /// storing parameters of linear models of the specified type. + static std::unique_ptr createParameters(const std::string &name); + + /// \brief Return the names of all linear models that can be created by one of the registered + /// makers. + static std::vector getMakerNames() { + return keys(getMakers()); + } + + virtual ~LinearModelFactory() = default; + + protected: + /// \brief Register a maker able to create linear models of type \p name. + explicit LinearModelFactory(const std::string & name); + + private: + virtual LinearModelBase * make(const Geometry_ &, const LinearModelParametersBase &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + + static std::map < std::string, LinearModelFactory * > & getMakers() { + static std::map < std::string, LinearModelFactory * > makers_; + return makers_; + } +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of LinearModelFactory able to create instances of T (a concrete subclass of +/// LinearModelBase). Passes Geometry to the constructor of T. +template +class LinearModelMaker : public LinearModelFactory { + private: + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericLinearModelParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + public: + typedef Geometry Geometry_; + + explicit LinearModelMaker(const std::string & name) : LinearModelFactory(name) {} + + LinearModelBase * make(const Geometry_ & geom, + const LinearModelParametersBase & parameters) override { + Log::trace() << "LinearModelBase::make starting" << std::endl; + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return new T(geom, + parametersOrConfiguration::value>(stronglyTypedParameters)); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } +}; + +// ----------------------------------------------------------------------------- + +template +LinearModelFactory::LinearModelFactory(const std::string & name) { + if (getMakers().find(name) != getMakers().end()) { + throw std::runtime_error(name + " already registered in the linear model factory."); + } + getMakers()[name] = this; +} + +// ----------------------------------------------------------------------------- + +template +LinearModelBase * LinearModelFactory::create(const Geometry_ & geom, + const LinearModelParametersBase & parameters) { + Log::trace() << "LinearModelFactory::create starting" << std::endl; + const std::string &id = parameters.name.value().value(); + typename std::map*>::iterator + jerr = getMakers().find(id); + if (jerr == getMakers().end()) { + throw std::runtime_error(id + " does not exist in the linear model factory"); + } + LinearModelBase * ptr = jerr->second->make(geom, parameters); + Log::trace() << "LinearModelFactory::create done" << std::endl; + return ptr; +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr LinearModelFactory::createParameters( + const std::string &name) { + Log::trace() << "LinearModelFactory::createParameters starting" << std::endl; + typename std::map*>::iterator it = getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in the linear model factory"); + } + return it->second->makeParameters(); +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_GENERIC_LINEARMODELBASE_H_ diff --git a/src/oops/generic/LinearModelId.h b/src/oops/generic/LinearModelId.h deleted file mode 100644 index a51476a28..000000000 --- a/src/oops/generic/LinearModelId.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_GENERIC_LINEARMODELID_H_ -#define OOPS_GENERIC_LINEARMODELID_H_ - -#include -#include - -#include "oops/base/LinearModelBase.h" -#include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/ModelAuxControl.h" -#include "oops/interface/ModelAuxIncrement.h" -#include "oops/interface/State.h" -#include "oops/util/Duration.h" -#include "oops/util/Logger.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" -#include "oops/util/Timer.h" - -namespace eckit { - class Configuration; -} - -namespace oops { - -/// Encapsulates the linear forecast model. -/*! - * Generic implementation of the identity linear model. - */ - -// ----------------------------------------------------------------------------- - -template -class LinearModelId : public LinearModelBase { - typedef typename MODEL::Increment Increment_; - typedef typename MODEL::Geometry Geometry_; - typedef typename MODEL::ModelAuxControl ModelAux_; - typedef typename MODEL::ModelAuxIncrement ModelAuxIncr_; - typedef typename MODEL::State State_; - - public: - static const std::string classname() {return "oops::LinearModelId";} - - LinearModelId(const Geometry_ &, const eckit::Configuration &); - ~LinearModelId(); - -// Set the linearization trajectory - void setTrajectory(const State_ &, State_ &, const ModelAux_ &) override; - -// Run the TL forecast - void initializeTL(Increment_ &) const override; - void stepTL(Increment_ & dx, const ModelAuxIncr_ &) const override; - void finalizeTL(Increment_ &) const override; - -// Run the AD forecast: - void initializeAD(Increment_ &) const override; - void stepAD(Increment_ & dx, ModelAuxIncr_ &) const override; - void finalizeAD(Increment_ &) const override; - -// Information and diagnostics - const util::Duration & timeResolution() const override {return tstep_;} - const oops::Variables & variables() const override {return vars_;} - void print(std::ostream &) const override {} - - private: - const util::Duration tstep_; - const Variables vars_; -}; - -// ============================================================================= - -template -LinearModelId::LinearModelId(const Geometry_ &, const eckit::Configuration & tlConf) - : tstep_(util::Duration(tlConf.getString("tstep"))), vars_() -{ - Log::trace() << "LinearModelId::LinearModelId" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -LinearModelId::~LinearModelId() { - Log::trace() << "LinearModelId::~LinearModelId" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::setTrajectory(const State_ &, State_ &, const ModelAux_ &) { - Log::trace() << "LinearModelId::setTrajectory" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::initializeTL(Increment_ &) const { - Log::trace() << "LinearModelId::initializeTL" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::stepTL(Increment_ & dx, const ModelAuxIncr_ &) const { - dx.updateTime(tstep_); - Log::trace() << "LinearModelId::stepTL" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::finalizeTL(Increment_ &) const { - Log::trace() << "LinearModelId::finalizeTL" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::initializeAD(Increment_ &) const { - Log::trace() << "LinearModelId::initializeAD" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::stepAD(Increment_ & dx, ModelAuxIncr_ &) const { - dx.updateTime(-tstep_); - Log::trace() << "LinearModelId::stepAD" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void LinearModelId::finalizeAD(Increment_ &) const { - Log::trace() << "LinearModelId::finalizeAD" << std::endl; -} - -// ----------------------------------------------------------------------------- -} // namespace oops - -#endif // OOPS_GENERIC_LINEARMODELID_H_ diff --git a/src/oops/base/LocalizationBase.h b/src/oops/generic/LocalizationBase.h similarity index 51% rename from src/oops/base/LocalizationBase.h rename to src/oops/generic/LocalizationBase.h index 40b83d654..4de3786b0 100644 --- a/src/oops/base/LocalizationBase.h +++ b/src/oops/generic/LocalizationBase.h @@ -9,8 +9,8 @@ * does it submit to any jurisdiction. */ -#ifndef OOPS_BASE_LOCALIZATIONBASE_H_ -#define OOPS_BASE_LOCALIZATIONBASE_H_ +#ifndef OOPS_GENERIC_LOCALIZATIONBASE_H_ +#define OOPS_GENERIC_LOCALIZATIONBASE_H_ #include #include @@ -18,10 +18,9 @@ #include -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/mpi/mpi.h" -#include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" @@ -32,111 +31,49 @@ namespace eckit { namespace oops { // ----------------------------------------------------------------------------- -/// Model-space localization base class +/// \brief Base class for generic implementations of model-space localization. +/// Use this class as a base class for generic implementations, +/// and interface::LocalizationBase as a base class for MODEL-specific implementations. +/// +/// Note: generic implementations need to provide a constructor with the following signature: +/// +/// LocalizationBase(const Geometry &, +/// const eckit::Configuration &); template class LocalizationBase : public util::Printable, private boost::noncopyable { - typedef Increment Increment_; + typedef Increment Increment_; public: LocalizationBase() = default; virtual ~LocalizationBase() = default; - /// 4D localization with the same localization for time blocks - virtual void randomize(Increment_ &) const; - virtual void multiply(Increment_ &) const; - - protected: - virtual void doRandomize(Increment_ &) const = 0; - virtual void doMultiply(Increment_ &) const = 0; - - private: - virtual void print(std::ostream &) const = 0; + /// Randomize \p dx and apply 3D localization + virtual void randomize(Increment_ & dx) const = 0; + /// Apply 3D localization to \p dx + virtual void multiply(Increment_ & dx) const = 0; }; // ----------------------------------------------------------------------------- -/// Randomize -template -void LocalizationBase::randomize(Increment_ & dx) const { - Log::trace() << "LocalizationBase::randomize starting" << std::endl; - const eckit::mpi::Comm & comm = dx.timeComm(); - static int tag = 23456; - size_t nslots = comm.size(); - int mytime = comm.rank(); - - if (mytime > 0) { - util::DateTime dt = dx.validTime(); // Save original time value - dx.zero(); - oops::mpi::receive(comm, dx, 0, tag); - dx.updateTime(dt - dx.validTime()); // Set time back to original value - } else { - // Apply 3D localization - this->doRandomize(dx); - - // Copy result to all timeslots - for (size_t jj = 1; jj < nslots; ++jj) { - oops::mpi::send(comm, dx, jj, tag); - } - } - ++tag; - - Log::trace() << "LocalizationBase::randomize done" << std::endl; -} - -// ----------------------------------------------------------------------------- -/// Multiply -template -void LocalizationBase::multiply(Increment_ & dx) const { - Log::trace() << "LocalizationBase::multiply starting" << std::endl; - const eckit::mpi::Comm & comm = dx.timeComm(); - static int tag = 23456; - size_t nslots = comm.size(); - int mytime = comm.rank(); - - if (mytime > 0) { - util::DateTime dt = dx.validTime(); // Save original time value - oops::mpi::send(comm, dx, 0, tag); - dx.zero(); - oops::mpi::receive(comm, dx, 0, tag); - dx.updateTime(dt - dx.validTime()); // Set time back to original value - } else { - // Sum over timeslots - for (size_t jj = 1; jj < nslots; ++jj) { - Increment_ dxtmp(dx); - oops::mpi::receive(comm, dxtmp, jj, tag); - dx.axpy(1.0, dxtmp, false); - } - - // Apply 3D localization - this->doMultiply(dx); - - // Copy result to all timeslots - for (size_t jj = 1; jj < nslots; ++jj) { - oops::mpi::send(comm, dx, jj, tag); - } - } - ++tag; - - Log::trace() << "LocalizationBase::multiply done" << std::endl; -} - -// ============================================================================= /// Localization Factory template class LocalizationFactory { typedef Geometry Geometry_; public: + /// \brief Create and return a new Localization. + /// + /// The Localization type is determined by the "localization method" entry of + /// \p config. static std::unique_ptr> create(const Geometry_ &, - const util::DateTime &, - const eckit::Configuration &); + const eckit::Configuration & config); virtual ~LocalizationFactory() = default; protected: - explicit LocalizationFactory(const std::string &); + /// \brief Register a maker able to create Localizations of type \p name. + explicit LocalizationFactory(const std::string & name); private: - virtual LocalizationBase * make(const Geometry_ &, - const util::DateTime &, - const eckit::Configuration &) = 0; + virtual std::unique_ptr> make(const Geometry_ &, + const eckit::Configuration &) = 0; static std::map < std::string, LocalizationFactory * > & getMakers() { static std::map < std::string, LocalizationFactory * > makers_; return makers_; @@ -144,19 +81,19 @@ class LocalizationFactory { }; // ----------------------------------------------------------------------------- - +/// \brief A subclass of LocalizationFactory able to create instances of T (a concrete +/// subclass of LocalizationBase). Passes Geometry to the constructor of T. template class LocalizationMaker : public LocalizationFactory { typedef Geometry Geometry_; - virtual LocalizationBase * make(const Geometry_ & geometry, - const util::DateTime & time, - const eckit::Configuration & conf) - { return new T(geometry, time, conf); } + std::unique_ptr> make(const Geometry_ & geometry, + const eckit::Configuration & conf) override + { return std::make_unique(geometry, conf); } public: explicit LocalizationMaker(const std::string & name) : LocalizationFactory(name) {} }; -// ============================================================================= +// ----------------------------------------------------------------------------- template LocalizationFactory::LocalizationFactory(const std::string & name) { @@ -171,7 +108,6 @@ LocalizationFactory::LocalizationFactory(const std::string & name) { template std::unique_ptr> LocalizationFactory::create(const Geometry_ & geometry, - const util::DateTime & time, const eckit::Configuration & conf) { Log::trace() << "LocalizationBase::create starting" << std::endl; const std::string id = conf.getString("localization method"); @@ -187,7 +123,7 @@ LocalizationFactory::create(const Geometry_ & geometry, } throw std::runtime_error(id + " does not exist in localization factory."); } - std::unique_ptr> ptr(jloc->second->make(geometry, time, conf)); + std::unique_ptr> ptr(jloc->second->make(geometry, conf)); Log::trace() << "LocalizationBase::create done" << std::endl; return ptr; } @@ -196,4 +132,4 @@ LocalizationFactory::create(const Geometry_ & geometry, } // namespace oops -#endif // OOPS_BASE_LOCALIZATIONBASE_H_ +#endif // OOPS_GENERIC_LOCALIZATIONBASE_H_ diff --git a/src/oops/base/ModelBase.h b/src/oops/generic/ModelBase.h similarity index 69% rename from src/oops/base/ModelBase.h rename to src/oops/generic/ModelBase.h index 539c0792b..75ed0a9c4 100644 --- a/src/oops/base/ModelBase.h +++ b/src/oops/generic/ModelBase.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_BASE_MODELBASE_H_ -#define OOPS_BASE_MODELBASE_H_ +#ifndef OOPS_GENERIC_MODELBASE_H_ +#define OOPS_GENERIC_MODELBASE_H_ #include #include @@ -16,9 +16,9 @@ #include #include -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/AssociativeContainers.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -38,21 +38,35 @@ namespace oops { // ----------------------------------------------------------------------------- -/// \brief Base class for the forecasting model. -/// Defines the interfaces for a forecast model. +/// \brief Base class for generic implementations of the forecasting models. /// Use this class as a base class for generic implementations, -/// and ModelBase as a base calss for MODEL-specific implementations. +/// and interface::ModelBase as a base class for MODEL-specific implementations. +/// +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of ModelParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// ModelBase(const Geometry_ &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of ModelParametersBase +/// holding the settings of the model in question. The implementation of the ModelBase interface +/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with +/// the following signature: +/// +/// ModelBase(const Geometry_ &, const Parameters_ &); +/// template -class GenericModelBase : public util::Printable, - private boost::noncopyable { +class ModelBase : public util::Printable, + private boost::noncopyable { typedef ModelAuxControl ModelAux_; typedef State State_; public: - static const std::string classname() {return "oops::GenericModelBase";} + static const std::string classname() {return "oops::ModelBase";} - GenericModelBase() = default; - virtual ~GenericModelBase() = default; + ModelBase() = default; + virtual ~ModelBase() = default; /// \brief Forecast initialization, called before every forecast run virtual void initialize(State_ &) const = 0; @@ -72,40 +86,6 @@ class GenericModelBase : public util::Printable, virtual void print(std::ostream &) const = 0; }; - -/// \brief Base class for MODEL-specific implementations of Model class. -/// The complete interface that needs to be implemented is described in GenericModelBase. -/// ModelBase overrides GenericModelBase methods to pass MODEL-specific implementations -/// of State and ModelAuxControl to the MODEL-specific implementation of Model. -template -class ModelBase : public GenericModelBase { - typedef typename MODEL::ModelAuxControl ModelAux_; - typedef typename MODEL::State State_; - - public: - static const std::string classname() {return "oops::ModelBase";} - - ModelBase() = default; - virtual ~ModelBase() = default; - - /// Overrides for ModelBase classes, passing MODEL-specific classes to the - /// MODEL-specific implementations of Model - void initialize(State & xx) const final - { this->initialize(xx.state()); } - void step(State & xx, const ModelAuxControl & modelaux) const final - { this->step(xx.state(), modelaux.modelauxcontrol()); } - void finalize(State & xx) const final - { this->finalize(xx.state()); } - - /// \brief Forecast initialization, called before every forecast run - virtual void initialize(State_ &) const = 0; - /// \brief Forecast "step", called during forecast run; updates state to the next time - virtual void step(State_ &, const ModelAux_ &) const = 0; - /// \brief Forecast finalization; called after each forecast run - virtual void finalize(State_ &) const = 0; -}; - - // ============================================================================= template @@ -172,8 +152,8 @@ class ModelFactory { /// The model's type is determined by the \c name attribute of \p parameters. /// \p parameters must be an instance of the subclass of ModelParametersBase /// associated with that model type, otherwise an exception will be thrown. - static GenericModelBase * create(const Geometry_ &, - const ModelParametersBase & parameters); + static ModelBase * create(const Geometry_ &, + const ModelParametersBase & parameters); /// \brief Create and return an instance of the subclass of ModelParametersBase /// storing parameters of models of the specified type. @@ -191,7 +171,7 @@ class ModelFactory { explicit ModelFactory(const std::string & name); private: - virtual GenericModelBase * make(const Geometry_ &, const ModelParametersBase &) = 0; + virtual ModelBase * make(const Geometry_ &, const ModelParametersBase &) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -204,36 +184,7 @@ class ModelFactory { // ----------------------------------------------------------------------------- /// \brief A subclass of ModelFactory able to create instances of T (a concrete subclass of -/// GenericModelBase). Passes Geometry to the generic implementation of Model. -template -class GenericModelMaker : public ModelFactory { - private: - /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as - /// GenericModelParameters. - typedef TParameters_IfAvailableElseFallbackType_t Parameters_; - - public: - typedef Geometry Geometry_; - - explicit GenericModelMaker(const std::string & name) : ModelFactory(name) {} - - GenericModelBase * make(const Geometry_ & geom, - const ModelParametersBase & parameters) override { - Log::trace() << "ModelBase::make starting" << std::endl; - const auto &stronglyTypedParameters = dynamic_cast(parameters); - return new T(geom, - parametersOrConfiguration::value>(stronglyTypedParameters)); - } - - std::unique_ptr makeParameters() const override { - return boost::make_unique(); - } -}; - -// ----------------------------------------------------------------------------- - -/// \brief A subclass of ModelFactory able to create instances of T (a concrete subclass of -/// ModelBase). Passes MODEL::Geometry to the MODEL-specific implementation of Model. +/// ModelBase). Passes Geometry to the constructor of T. template class ModelMaker : public ModelFactory { private: @@ -246,10 +197,11 @@ class ModelMaker : public ModelFactory { explicit ModelMaker(const std::string & name) : ModelFactory(name) {} - ModelBase * make(const Geometry_ & geom, const ModelParametersBase & parameters) override { + ModelBase * make(const Geometry_ & geom, + const ModelParametersBase & parameters) override { Log::trace() << "ModelBase::make starting" << std::endl; const auto &stronglyTypedParameters = dynamic_cast(parameters); - return new T(geom.geometry(), + return new T(geom, parametersOrConfiguration::value>(stronglyTypedParameters)); } @@ -258,7 +210,6 @@ class ModelMaker : public ModelFactory { } }; - // ----------------------------------------------------------------------------- template @@ -272,7 +223,7 @@ ModelFactory::ModelFactory(const std::string & name) { // ----------------------------------------------------------------------------- template -GenericModelBase * ModelFactory::create(const Geometry_ & geom, +ModelBase * ModelFactory::create(const Geometry_ & geom, const ModelParametersBase & parameters) { Log::trace() << "ModelFactory::create starting" << std::endl; const std::string &id = parameters.name.value().value(); @@ -281,7 +232,7 @@ GenericModelBase * ModelFactory::create(const Geometry_ & geom, if (jerr == getMakers().end()) { throw std::runtime_error(id + " does not exist in the model factory"); } - GenericModelBase * ptr = jerr->second->make(geom, parameters); + ModelBase * ptr = jerr->second->make(geom, parameters); Log::trace() << "ModelFactory::create done" << std::endl; return ptr; } @@ -303,4 +254,4 @@ std::unique_ptr ModelFactory::createParameters( } // namespace oops -#endif // OOPS_BASE_MODELBASE_H_ +#endif // OOPS_GENERIC_MODELBASE_H_ diff --git a/src/oops/generic/ObsErrorBase.h b/src/oops/generic/ObsErrorBase.h new file mode 100644 index 000000000..19e8af2c0 --- /dev/null +++ b/src/oops/generic/ObsErrorBase.h @@ -0,0 +1,227 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_GENERIC_OBSERRORBASE_H_ +#define OOPS_GENERIC_OBSERRORBASE_H_ + +#include +#include +#include +#include + +#include +#include "eckit/config/Configuration.h" + +#include "oops/base/ObsVector.h" +#include "oops/interface/ObsSpace.h" +#include "oops/util/AssociativeContainers.h" +#include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/PolymorphicParameter.h" +#include "oops/util/Printable.h" + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief Base class for generic implementations of observation error covariance matrices. +/// +/// Use this class as a base class for generic implementations, +/// and interface::ObsErrorBase as a base class for OBS-specific implementations. +/// +/// Note: each generic implementation should typedef `Parameters_` to the name of a subclass of +/// ObsErrorParametersBase holding its configuration settings and provide a constructor with the +/// following signature: +/// +/// ObsErrorBase(const Parameters_ &, ObsSpace &); +template +class ObsErrorBase : public util::Printable, + private boost::noncopyable { + typedef ObsVector ObsVector_; + typedef ObsSpace ObsSpace_; + + public: + ObsErrorBase() = default; + virtual ~ObsErrorBase() = default; + +/// Multiply a Departure \p dy by \f$R\f$. + virtual void multiply(ObsVector_ & dy) const = 0; +/// Multiply a Departure \p dy by \f$R^{-1}\f$. + virtual void inverseMultiply(ObsVector_ & dy) const = 0; + +/// Generate random perturbation in \p dy. + virtual void randomize(ObsVector_ & dy) const = 0; + +/// Save obs errors to the group \p name. + virtual void save(const std::string &name) const = 0; + +/// Return a copy of obs error std. dev. If this ObsVector_ is modified (e.g. by obs filters), +/// it should be passed back to update() to ensure the covariance matrix stays consistent. + virtual ObsVector_ obserrors() const = 0; + +/// Set the diagonal of the covariance matrix to \p stddev squared. + virtual void update(const ObsVector_ &stddev) = 0; + +/// Return the vector of inverse obs error variances. + virtual ObsVector_ inverseVariance() const = 0; + +/// Get mean error for Jo table. + virtual double getRMSE() const = 0; + + private: + virtual void print(std::ostream &) const = 0; +}; + +// ============================================================================= + +template +class ObsErrorFactory; + +// ----------------------------------------------------------------------------- + +/// \brief Configuration parameters of an implementation of an observation error covariance matrix +/// model. +class ObsErrorParametersBase : public Parameters { + OOPS_ABSTRACT_PARAMETERS(ObsErrorParametersBase, Parameters) + public: + /// \brief Name of the covariance model. + Parameter model{"covariance model", "diagonal", this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// ObsErrorParametersBase. +template +class ObsErrorParametersWrapper : public Parameters { + OOPS_CONCRETE_PARAMETERS(ObsErrorParametersWrapper, Parameters) + public: + /// After deserialization, holds an instance of a subclass of ObsErrorParametersBase controlling + /// the behavior of an observation error covariance matrix model. The type of the subclass is + /// determined by the value of the "covariance model" key in the Configuration object from which + /// this object is deserialized. + PolymorphicParameter> + obsErrorParameters{"covariance model", "diagonal", this}; +}; + +// ----------------------------------------------------------------------------- + +/// A factory creating instances of concrete subclasses of ObsErrorBase. +template +class ObsErrorFactory { + typedef ObsErrorBase ObsErrorBase_; + typedef ObsSpace ObsSpace_; + + public: + /// \brief Create and return a new observation error covariance matrix model. + /// + /// The model's type is determined by the `covariance model` attribute of \p params. + /// \p params must be an instance of the subclass of ObsErrorParametersBase + /// associated with that model type, otherwise an exception will be thrown. + static std::unique_ptr create(const ObsErrorParametersBase ¶ms, + const ObsSpace_ &); + + /// \brief Create and return an instance of the subclass of ObsErrorParametersBase + /// storing parameters of covariance matrix models of the specified type. + static std::unique_ptr createParameters(const std::string &model); + + /// \brief Return the names of all models that can be created by one of the registered makers. + static std::vector getMakerNames() { + return keys(getMakers()); + } + + virtual ~ObsErrorFactory() = default; + + protected: + /// \brief Register a maker able to create covariance matrix models of type \p model. + explicit ObsErrorFactory(const std::string &model); + + private: + virtual std::unique_ptr make(const ObsErrorParametersBase &, + const ObsSpace_ &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + + static std::map < std::string, ObsErrorFactory * > & getMakers() { + static std::map < std::string, ObsErrorFactory * > makers_; + return makers_; + } +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of ObsErrorFactory able to create instances of T (a concrete subclass of +/// ObsErrorBase). Passes ObsSpace to the constructor of T. +template +class ObsErrorMaker : public ObsErrorFactory { + typedef ObsErrorBase ObsErrorBase_; + typedef ObsSpace ObsSpace_; + typedef typename T::Parameters_ Parameters_; + + std::unique_ptr make(const ObsErrorParametersBase & parameters, + const ObsSpace_ & obs) override { + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return std::make_unique(stronglyTypedParameters, obs); + } + + std::unique_ptr makeParameters() const override { + return std::make_unique(); + } + + public: + explicit ObsErrorMaker(const std::string & name) : ObsErrorFactory(name) {} +}; + +// ============================================================================= + +template +ObsErrorFactory::ObsErrorFactory(const std::string & name) { + if (getMakers().find(name) != getMakers().end()) { + throw std::runtime_error(name + " already registered in obs error factory."); + } + getMakers()[name] = this; +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr> +ObsErrorFactory::create(const ObsErrorParametersBase & params, const ObsSpace_ & obs) { + Log::trace() << "ObsErrorFactory::create starting" << std::endl; + const std::string &id = params.model; + Log::trace() << "ObsError matrix type is: " << id << std::endl; + typename std::map*>::iterator + jerr = getMakers().find(id); + if (jerr == getMakers().end()) { + throw std::runtime_error(id + " does not exist in obs error factory."); + } + std::unique_ptr> ptr(jerr->second->make(params, obs)); + Log::trace() << "ObsErrorFactory::create done" << std::endl; + return ptr; +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr ObsErrorFactory::createParameters( + const std::string &name) { + Log::trace() << "ObsErrorFactory::createParameters starting" << std::endl; + typename std::map*>::iterator it = getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in obs error factory"); + } + return it->second->makeParameters(); +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_GENERIC_OBSERRORBASE_H_ diff --git a/src/oops/generic/ObsErrorDiag.h b/src/oops/generic/ObsErrorDiag.h index c767263b5..561c3e19a 100644 --- a/src/oops/generic/ObsErrorDiag.h +++ b/src/oops/generic/ObsErrorDiag.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -15,9 +16,9 @@ #include #include "eckit/config/Configuration.h" -#include "oops/base/ObsErrorBase.h" +#include "oops/base/ObsVector.h" +#include "oops/generic/ObsErrorBase.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" @@ -25,8 +26,8 @@ namespace oops { /// \brief Parameters for diagonal obs errors -class ObsErrorDiagParameters : public Parameters { - OOPS_CONCRETE_PARAMETERS(ObsErrorDiagParameters, Parameters) +class ObsErrorDiagParameters : public ObsErrorParametersBase { + OOPS_CONCRETE_PARAMETERS(ObsErrorDiagParameters, ObsErrorParametersBase) public: /// perturbation amplitude multiplier Parameter pert{"random amplitude", 1.0, this}; @@ -40,10 +41,14 @@ class ObsErrorDiag : public ObsErrorBase { typedef ObsVector ObsVector_; public: - ObsErrorDiag(const eckit::Configuration &, const ObsSpace_ &); + /// The type of parameters passed to the constructor. + /// This typedef is used by the ObsErrorFactory. + typedef ObsErrorDiagParameters Parameters_; + + ObsErrorDiag(const Parameters_ &, const ObsSpace_ &); /// Update after obs errors potentially changed - void update() override; + void update(const ObsVector_ &) override; /// Multiply a Departure by \f$R\f$ void multiply(ObsVector_ &) const override; @@ -61,36 +66,35 @@ class ObsErrorDiag : public ObsErrorBase { double getRMSE() const override {return stddev_.rms();} /// Get obs errors std deviation - ObsVector_ & obserrors() override {return stddev_;} - const ObsVector_ & obserrors() const override {return stddev_;} + ObsVector_ obserrors() const override {return stddev_;} -/// Return inverseVariance - const ObsVector_ & inverseVariance() const override {return inverseVariance_;} - - protected: - ObsVector_ stddev_; - ObsVector_ inverseVariance_; +/// Get inverseVariance + ObsVector_ inverseVariance() const override {return inverseVariance_;} private: void print(std::ostream &) const override; - ObsErrorDiagParameters options_; + ObsVector_ stddev_; + ObsVector_ inverseVariance_; + Parameters_ options_; }; // ============================================================================= template -ObsErrorDiag::ObsErrorDiag(const eckit::Configuration & conf, const ObsSpace_ & obsgeom) - : stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) +ObsErrorDiag::ObsErrorDiag(const ObsErrorDiagParameters & options, const ObsSpace_ & obsgeom) + : stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom), options_(options) { - options_.deserialize(conf); - this->update(); + inverseVariance_ = stddev_; + inverseVariance_ *= stddev_; + inverseVariance_.invert(); Log::trace() << "ObsErrorDiag:ObsErrorDiag constructed nobs = " << stddev_.nobs() << std::endl; } // ----------------------------------------------------------------------------- template -void ObsErrorDiag::update() { +void ObsErrorDiag::update(const ObsVector_ & obserr) { + stddev_ = obserr; inverseVariance_ = stddev_; inverseVariance_ *= stddev_; inverseVariance_.invert(); diff --git a/src/oops/base/ObsFilterBase.h b/src/oops/generic/ObsFilterBase.h similarity index 69% rename from src/oops/base/ObsFilterBase.h rename to src/oops/generic/ObsFilterBase.h index bcc9fa580..ee46032a0 100644 --- a/src/oops/base/ObsFilterBase.h +++ b/src/oops/generic/ObsFilterBase.h @@ -5,25 +5,20 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_BASE_OBSFILTERBASE_H_ -#define OOPS_BASE_OBSFILTERBASE_H_ +#ifndef OOPS_GENERIC_OBSFILTERBASE_H_ +#define OOPS_GENERIC_OBSFILTERBASE_H_ #include #include #include #include -#include #include -#include "oops/base/ObsFilterParametersBase.h" #include "oops/base/Variables.h" -#include "oops/interface/GeoVaLs.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsDiagnostics.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" +#include "oops/generic/ObsFilterParametersBase.h" #include "oops/util/AssociativeContainers.h" +#include "oops/util/Logger.h" #include "oops/util/parameters/ConfigurationParameter.h" #include "oops/util/parameters/HasParameters_.h" #include "oops/util/parameters/OptionalParameter.h" @@ -34,9 +29,33 @@ namespace oops { -/// Base class for QC filters applied to observations -// ----------------------------------------------------------------------------- +template class GeoVaLs; +template class ObsDiagnostics; +template class ObsSpace; +template class ObsVector; +template class ObsDataVector; + +/// \brief Base class for generic implementations of filters processing observations. +/// +/// Use this class as a base class for generic implementations +/// and interface::ObsFilterBase as a base class for OBS-specific implementations. +/// +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of ObsFilterParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const eckit::Configuration &, +/// ObsDataPtr_, ObsDataPtr_); +/// +/// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase +/// holding the settings of the filter in question. The implementation of the ObsFilter interface +/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with +/// the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const Parameters_ &, +/// ObsDataPtr_, ObsDataPtr_); template class ObsFilterBase : public util::Printable, private boost::noncopyable { @@ -48,15 +67,30 @@ class ObsFilterBase : public util::Printable, ObsFilterBase() {} virtual ~ObsFilterBase() {} - virtual void preProcess() const = 0; - virtual void priorFilter(const GeoVaLs_ &) const = 0; - virtual void postFilter(const ObsVector_ &, const ObsDiags_ &) const = 0; + /// \brief Perform any observation processing steps that do not require access to GeoVaLs or + /// outputs produced by the observation operator. + virtual void preProcess() = 0; + + /// \brief Perform any observation processing steps that require access to GeoVaLs, but not to + /// outputs produced by the observation operator. + virtual void priorFilter(const GeoVaLs_ &gv) = 0; + /// \brief Perform any observation processing steps that require access to + /// outputs produced by the observation operator. + /// + /// \param ov + /// Model equivalents produced by the observation operator. + /// \param bv + /// Bias of departure produced by the observation operator. + /// \param dv + /// Observation diagnostics produced by the observation operator. + virtual void postFilter(const ObsVector_ &ov, const ObsVector_ &bv, const ObsDiags_ &dv) = 0; + + /// \brief Return the list of GeoVaLs required by this filter. virtual Variables requiredVars() const = 0; - virtual Variables requiredHdiagnostics() const = 0; - private: - virtual void print(std::ostream &) const = 0; + /// \brief Return the list of observation diagnostics required by this filter. + virtual Variables requiredHdiagnostics() const = 0; }; // ============================================================================= @@ -102,7 +136,7 @@ class ObsFilterParametersWrapper : public Parameters { // ============================================================================= -/// ObsFilter Factory +/// ObsFilter factory template class FilterFactory { typedef ObsSpace ObsSpace_; @@ -111,22 +145,14 @@ class FilterFactory { public: /// \brief Create and return a new observation filter. /// - /// The type of the filter is determined by the `Filter` attribute of \p parameters. \p params + /// The type of the filter is determined by the `filter` attribute of \p params. \p params /// must be an instance of the subclass of ObsFilterParametersBase associated with that filter, /// otherwise an exception will be thrown. - static std::shared_ptr> create(const ObsSpace_ &, + static std::unique_ptr> create(const ObsSpace_ &, const ObsFilterParametersBase & params, ObsDataPtr_ flags = ObsDataPtr_(), ObsDataPtr_ obserr = ObsDataPtr_()); - /// \brief Create and return a new observation filter. - /// - /// Deprecated overload taking a Configuration instead of an ObsFilterParametersBase. - static std::shared_ptr> create(const ObsSpace_ &, - const eckit::Configuration &, - ObsDataPtr_ flags = ObsDataPtr_(), - ObsDataPtr_ obserr = ObsDataPtr_()); - /// \brief Create and return an instance of the subclass of ObsFilterParametersBase /// storing parameters of observation filters of the specified type. static std::unique_ptr createParameters( @@ -145,8 +171,9 @@ class FilterFactory { explicit FilterFactory(const std::string &name); private: - virtual ObsFilterBase * make(const ObsSpace_ &, const ObsFilterParametersBase &, - ObsDataPtr_ &, ObsDataPtr_ &) = 0; + virtual std::unique_ptr> make(const ObsSpace_ &, + const ObsFilterParametersBase &, + ObsDataPtr_ &, ObsDataPtr_ &) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -167,17 +194,19 @@ class FilterMaker : public FilterFactory { typedef ObsSpace ObsSpace_; template using ObsDataPtr_ = std::shared_ptr >; - ObsFilterBase * make(const ObsSpace_ & os, const ObsFilterParametersBase & params, - ObsDataPtr_ & flags, ObsDataPtr_ & obserr) override { + std::unique_ptr> make(const ObsSpace_ & os, + const ObsFilterParametersBase & params, + ObsDataPtr_ & flags, + ObsDataPtr_ & obserr) override { const auto &stronglyTypedParams = dynamic_cast(params); - return new T(os, + return std::make_unique(os, parametersOrConfiguration::value>(stronglyTypedParams), flags, obserr); } std::unique_ptr makeParameters() const override { - return boost::make_unique(); + return std::make_unique(); } public: @@ -197,7 +226,7 @@ FilterFactory::FilterFactory(const std::string & name) { // ----------------------------------------------------------------------------- template -std::shared_ptr> +std::unique_ptr> FilterFactory::create(const ObsSpace_ & os, const ObsFilterParametersBase & params, ObsDataPtr_ flags, ObsDataPtr_ obserr) { Log::trace() << "FilterFactory::create starting" << std::endl; @@ -213,24 +242,13 @@ FilterFactory::create(const ObsSpace_ & os, const ObsFilterParametersBase & } throw std::runtime_error(id + " does not exist in obs filter factory."); } - std::shared_ptr> ptr(jloc->second->make(os, params, flags, obserr)); + std::unique_ptr> ptr(jloc->second->make(os, params, flags, obserr)); Log::trace() << "FilterFactory::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- -template -std::shared_ptr> -FilterFactory::create(const ObsSpace_ & os, const eckit::Configuration & conf, - ObsDataPtr_ flags, ObsDataPtr_ obserr) { - ObsFilterParametersWrapper parameters; - parameters.validateAndDeserialize(conf); - return create(os, parameters.filterParameters, flags, obserr); -} - -// ----------------------------------------------------------------------------- - template std::unique_ptr FilterFactory::createParameters(const std::string &name) { @@ -246,4 +264,4 @@ FilterFactory::createParameters(const std::string &name) { } // namespace oops -#endif // OOPS_BASE_OBSFILTERBASE_H_ +#endif // OOPS_GENERIC_OBSFILTERBASE_H_ diff --git a/src/oops/base/ObsFilterParametersBase.h b/src/oops/generic/ObsFilterParametersBase.h similarity index 88% rename from src/oops/base/ObsFilterParametersBase.h rename to src/oops/generic/ObsFilterParametersBase.h index 801f8d0ae..80ef43080 100644 --- a/src/oops/base/ObsFilterParametersBase.h +++ b/src/oops/generic/ObsFilterParametersBase.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ -#define OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ +#ifndef OOPS_GENERIC_OBSFILTERPARAMETERSBASE_H_ +#define OOPS_GENERIC_OBSFILTERPARAMETERSBASE_H_ #include @@ -31,4 +31,4 @@ class ObsFilterParametersBase : public Parameters { } // namespace oops -#endif // OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ +#endif // OOPS_GENERIC_OBSFILTERPARAMETERSBASE_H_ diff --git a/src/oops/generic/PseudoModel.h b/src/oops/generic/PseudoModel.h index c9dcea409..21d59e9b5 100644 --- a/src/oops/generic/PseudoModel.h +++ b/src/oops/generic/PseudoModel.h @@ -11,10 +11,10 @@ #include #include -#include "oops/base/ModelBase.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" +#include "oops/generic/ModelBase.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -26,7 +26,7 @@ namespace oops { /// Generic implementation of the pseudo model (steps through time by reading states) template -class PseudoModel : public GenericModelBase { +class PseudoModel : public ModelBase { typedef Geometry Geometry_; typedef ModelAuxControl ModelAux_; typedef State State_; @@ -50,7 +50,7 @@ class PseudoModel : public GenericModelBase { const oops::Variables & variables() const override {return vars_;} private: - void print(std::ostream &) const override {} + void print(std::ostream &) const override; const util::Duration tstep_; const oops::Variables vars_; std::vector confs_; @@ -94,6 +94,11 @@ void PseudoModel::finalize(State_ & xx) const { // ----------------------------------------------------------------------------- +template +void PseudoModel::print(std::ostream & os) const { + os << "Pseudo model reading states from files with " << tstep_ << " time resolution"; +} + } // namespace oops #endif // OOPS_GENERIC_PSEUDOMODEL_H_ diff --git a/src/oops/generic/PseudoModelState4D.h b/src/oops/generic/PseudoModelState4D.h new file mode 100644 index 000000000..28ec6aaf5 --- /dev/null +++ b/src/oops/generic/PseudoModelState4D.h @@ -0,0 +1,116 @@ +/* + * (C) Copyright 2021- UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_GENERIC_PSEUDOMODELSTATE4D_H_ +#define OOPS_GENERIC_PSEUDOMODELSTATE4D_H_ + +#include +#include + +#include "oops/base/Geometry.h" +#include "oops/base/State.h" +#include "oops/base/State4D.h" +#include "oops/generic/ModelBase.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +/// Generic implementation of the pseudo model initialized with 4D State +/// (steps through time by stepping through states in 4D state) +template +class PseudoModelState4D : public ModelBase { + typedef Geometry Geometry_; + typedef ModelAuxControl ModelAux_; + typedef State State_; + typedef State4D State4D_; + + public: + static const std::string classname() {return "oops::PseudoModelState4D";} + + /// Initialize pseudo model with \p state4d - 4D state to loop through + /// in the model run and \p tstep - time resolution of the model + PseudoModelState4D(const State4D_ & state4d, + const util::Duration & tstep = util::Duration(0)); + + /// initialize forecast + void initialize(State_ &) const override; + /// one forecast step + void step(State_ &, const ModelAux_ &) const override; + /// finalize forecast + void finalize(State_ &) const override; + + /// model time step + const util::Duration & timeResolution() const override {return tstep_;} + + /// model variables + const oops::Variables & variables() const override {return vars_;} + + private: + void print(std::ostream &) const override; + + /// Reference to 4D state that is used in the model + const State4D_ & state4d_; + /// Model's time resolution + util::Duration tstep_; + /// Variables from 4D state + const oops::Variables vars_; + /// Index of the current state + mutable size_t currentstate_; +}; + +// ----------------------------------------------------------------------------- + +template +PseudoModelState4D::PseudoModelState4D(const State4D_ & state4d, + const util::Duration & tstep) + : state4d_(state4d), tstep_(tstep), vars_(state4d.variables()) { + const std::vector validTimes = state4d_.validTimes(); + if (validTimes.size() > 1) tstep_ = validTimes[1] - validTimes[0]; + Log::trace() << "PseudoModelState4D::PseudoModelState4D done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void PseudoModelState4D::initialize(State_ & xx) const { + currentstate_ = 0; + xx = state4d_[currentstate_]; + Log::trace() << "PseudoModelState4D::initialize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void PseudoModelState4D::step(State_ & xx, const ModelAux_ & merr) const { + Log::trace() << "PseudoModelState4D:step Starting " << std::endl; + xx = state4d_[currentstate_++]; + Log::trace() << "PseudoModelState4D::step done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void PseudoModelState4D::finalize(State_ & xx) const { + Log::trace() << "PseudoModelState4D::finalize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void PseudoModelState4D::print(std::ostream & os) const { + os << "Pseudo model stepping through 4D state with " << tstep_ << " time resolution"; +} + +} // namespace oops + +#endif // OOPS_GENERIC_PSEUDOMODELSTATE4D_H_ diff --git a/src/oops/generic/VerticalLocEV.h b/src/oops/generic/VerticalLocEV.h index 918064665..ec88f0d93 100644 --- a/src/oops/generic/VerticalLocEV.h +++ b/src/oops/generic/VerticalLocEV.h @@ -10,17 +10,20 @@ #include +#include #include #include #include "eckit/config/Configuration.h" -#include "oops/assimilation/Increment4D.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment4D.h" +#include "oops/base/IncrementEnsemble.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/LocalIncrement.h" #include "oops/generic/gc99.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/Increment.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/parameters/Parameter.h" @@ -51,6 +54,10 @@ class VerticalLocalizationParameters : public Parameters { RequiredParameter VertLocDist{"lengthscale", this}; // localization distance at which Gaspari-Cohn = 0 RequiredParameter VertLocUnits{"lengthscale units", this}; + // write eigen vectors to disk + Parameter writeEVs{"write eigen vectors", false, this}; + // read eigen vectors from disk + Parameter readEVs{"read eigen vectors", false, this}; }; // ---------------------------------------------------------------------------- @@ -59,13 +66,16 @@ class VerticalLocEV: public util::Printable, private util::ObjectCounter> { typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; + typedef Increment Increment_; typedef Increment4D Increment4D_; + typedef IncrementEnsemble IncrementEnsemble_; typedef IncrementEnsemble4D IncrementEnsemble4D_; + typedef State State_; public: static const std::string classname() {return "oops::VerticalLocEV";} - VerticalLocEV(const Geometry_ & , const eckit::Configuration &); + VerticalLocEV(const eckit::Configuration &, const State_ &); // modulate an increment void modulateIncrement(const Increment4D_ &, IncrementEnsemble4D_ &) const; @@ -92,25 +102,89 @@ class VerticalLocEV: public util::Printable, std::vector replicateEigenVector(const oops::Variables &, size_t) const; +// populate 3D increment array of eigen vectors + void populateIncrementEnsembleWithEVs(const Variables &); +// IO for eigen vectors + void writeEVsToDisk(const eckit::Configuration &) const; + void readEVsFromDisk(const Geometry_ &, const Variables &, const util::DateTime &, + const eckit::Configuration &); + private: void print(std::ostream &) const {} VerticalLocalizationParameters options_; Eigen::MatrixXd Evecs_; Eigen::VectorXd Evals_; size_t neig_; + std::unique_ptr sqrtVertLoc_; + // if true store EVs as explicit 3D fields + // TODO(frolovsa) depriciate 1D storage in the future + // once we see no issues with always using 3D + bool EVsStoredAs3D_ = true; // if true store EVs as explicit 3D fields }; // ----------------------------------------------------------------------------- template - VerticalLocEV::VerticalLocEV(const Geometry_ & geom, - const eckit::Configuration & conf) { + VerticalLocEV::VerticalLocEV(const eckit::Configuration & conf, + const State_ & x): sqrtVertLoc_() { // read vertical localization configuration options_.deserialize(conf); - // compute vertical corrleation matrix fo nLevs - Eigen::MatrixXd cov = computeCorrMatrix(geom); - // compute truncated correlation matrix - computeCorrMatrixEvec(cov); - neig_ = truncateEvecs(); + if (options_.readEVs) { + oops::Log::info() << "Reading precomputed vertical localization EVs from disk" << std::endl; + readEVsFromDisk(x.geometry(), x.variables(), x.validTime(), conf); + neig_ = sqrtVertLoc_->size(); + } else { + oops::Log::info() << "Computing vertical localization EVs from scratch" << std::endl; + // compute vertical corrleation matrix fo nLevs + Eigen::MatrixXd cov = computeCorrMatrix(x.geometry()); + // compute truncated correlation matrix + computeCorrMatrixEvec(cov); + neig_ = truncateEvecs(); + // convert EVs to 3D if needed + if (EVsStoredAs3D_) { + sqrtVertLoc_ = boost::make_unique + (x.geometry(), x.variables(), x.validTime(), neig_); + populateIncrementEnsembleWithEVs(x.variables()); + } + } + + if (options_.writeEVs) { writeEVsToDisk(conf); } + } + +// ----------------------------------------------------------------------------- +template + void VerticalLocEV::populateIncrementEnsembleWithEVs(const Variables & vars) { + const Geometry_ & geom = (*sqrtVertLoc_)[0].geometry(); + for (size_t ieig=0; ieig < neig_; ++ieig) { + std::vector EvecRepl = replicateEigenVector(vars, ieig); + (*sqrtVertLoc_)[ieig].ones(); + for (GeometryIterator_ gpi = geom.begin(); gpi != geom.end(); ++gpi) { + oops::LocalIncrement gp = (*sqrtVertLoc_)[ieig].getLocal(gpi); + gp *= EvecRepl; + (*sqrtVertLoc_)[ieig].setLocal(gp, gpi); + } + } + } + +// ----------------------------------------------------------------------------- +template + void VerticalLocEV::writeEVsToDisk(const eckit::Configuration & config) const { + eckit::LocalConfiguration outConfig(config, "list of eigen vectors to write"); + sqrtVertLoc_->write(outConfig); + } + +// ----------------------------------------------------------------------------- +template + void VerticalLocEV::readEVsFromDisk(const Geometry_ & geom, const Variables & vars, + const util::DateTime & tslot, + const eckit::Configuration & config) { + std::vector memberConfig; + config.get("list of eigen vectors to read", memberConfig); + sqrtVertLoc_ = boost::make_unique(geom, vars, tslot, memberConfig.size()); + + // Loop over all ensemble members + for (size_t jj = 0; jj < memberConfig.size(); ++jj) { + (*sqrtVertLoc_)[jj].read(memberConfig[jj]); + } } // ------------------------------------------------------------------------------------------------- @@ -200,6 +274,10 @@ template // ----------------------------------------------------------------------------- template bool VerticalLocEV::testTruncateEvecs(const Geometry_ & geom) { + // only do this test if we are computing EVs from scratch + // if reading from disk return true + if (options_.readEVs) {return true;} + // make reference solution Eigen::MatrixXd cov = computeCorrMatrix(geom); // only the lower traingle of cov is stored @@ -239,14 +317,19 @@ void VerticalLocEV::modulateIncrement(const Increment4D_ & incr, oops::Variables vars = incr[0].variables(); for (size_t ieig=0; ieig < neig_; ++ieig) { - std::vector EvecRepl = replicateEigenVector(vars, ieig); // modulate an increment for (size_t itime=0; itime < incr.size(); ++itime) { - const Geometry_ & geom = incr[itime].geometry(); - for (GeometryIterator_ gpi = geom.begin(); gpi != geom.end(); ++gpi) { - oops::LocalIncrement gp = incr[itime].getLocal(gpi); - gp *= EvecRepl; - incrsOut[ieig][itime].setLocal(gp, gpi); + if (EVsStoredAs3D_) { // if EVs stored as 3D use oops operation for schur_product + incrsOut[ieig][itime] = (*sqrtVertLoc_)[ieig]; + incrsOut[ieig][itime].schur_product_with(incr[itime]); + } else { // if EV is only available as a 1D column use grid eterator + std::vector EvecRepl = replicateEigenVector(vars, ieig); + const Geometry_ & geom = incr[itime].geometry(); + for (GeometryIterator_ gpi = geom.begin(); gpi != geom.end(); ++gpi) { + oops::LocalIncrement gp = incr[itime].getLocal(gpi); + gp *= EvecRepl; + incrsOut[ieig][itime].setLocal(gp, gpi); + } } } } @@ -261,17 +344,27 @@ Eigen::MatrixXd VerticalLocEV::modulateIncrement( // modulate an increment at grid point oops::Variables vars = incrs[0][0].variables(); - size_t nv = Evecs_.rows()*vars.size(); + size_t nv = 0; + std::vector EvecRepl; + if (EVsStoredAs3D_) { + nv = (*sqrtVertLoc_)[0].getLocal(gi).getVals().size(); + } else { + EvecRepl = replicateEigenVector(vars, 0); + nv = EvecRepl.size(); + } size_t nens = incrs.size(); Eigen::MatrixXd Z(nv, neig_*nens); std::vector etmp2(nv); size_t ii = 0; for (size_t iens=0; iens < nens; ++iens) { - oops::LocalIncrement gp = incrs[iens][itime].getLocal(gi); - etmp2 = gp.getVals(); + etmp2 = incrs[iens][itime].getLocal(gi).getVals(); for (size_t ieig=0; ieig < neig_; ++ieig) { - std::vector EvecRepl = replicateEigenVector(vars, ieig); + if (EVsStoredAs3D_) { + EvecRepl = (*sqrtVertLoc_)[ieig].getLocal(gi).getVals(); + } else { + EvecRepl = replicateEigenVector(vars, ieig); + } // modulate and assign for (size_t iv=0; iv < nv; ++iv) { Z(iv, ii) = EvecRepl[iv]*etmp2[iv]; @@ -281,6 +374,8 @@ Eigen::MatrixXd VerticalLocEV::modulateIncrement( } return Z; } + +// ----------------------------------------------------------------------------- template std::vector VerticalLocEV::replicateEigenVector( const oops::Variables & v, size_t ieig) const { diff --git a/src/oops/generic/instantiateLinearModelFactory.h b/src/oops/generic/instantiateLinearModelFactory.h new file mode 100644 index 000000000..480812a23 --- /dev/null +++ b/src/oops/generic/instantiateLinearModelFactory.h @@ -0,0 +1,22 @@ +/* + * (C) Copyright 2018-2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_GENERIC_INSTANTIATELINEARMODELFACTORY_H_ +#define OOPS_GENERIC_INSTANTIATELINEARMODELFACTORY_H_ + +#include "oops/generic/IdentityLinearModel.h" +#include "oops/generic/LinearModelBase.h" + +namespace oops { + +template void instantiateLinearModelFactory() { + static LinearModelMaker > makerIdentityLinearModel_("Identity"); +} + +} // namespace oops + +#endif // OOPS_GENERIC_INSTANTIATELINEARMODELFACTORY_H_ diff --git a/src/oops/generic/instantiateModelFactory.h b/src/oops/generic/instantiateModelFactory.h index b0211b03d..6c6dfd86e 100644 --- a/src/oops/generic/instantiateModelFactory.h +++ b/src/oops/generic/instantiateModelFactory.h @@ -8,15 +8,15 @@ #ifndef OOPS_GENERIC_INSTANTIATEMODELFACTORY_H_ #define OOPS_GENERIC_INSTANTIATEMODELFACTORY_H_ -#include "oops/base/ModelBase.h" #include "oops/generic/IdentityModel.h" +#include "oops/generic/ModelBase.h" #include "oops/generic/PseudoModel.h" namespace oops { template void instantiateModelFactory() { - static GenericModelMaker > makerIdentityModel_("Identity"); - static GenericModelMaker > makerPseudoModel_("PseudoModel"); + static ModelMaker > makerIdentityModel_("Identity"); + static ModelMaker > makerPseudoModel_("PseudoModel"); } } // namespace oops diff --git a/src/oops/generic/instantiateObsErrorFactory.h b/src/oops/generic/instantiateObsErrorFactory.h index b18e6b817..fd2cc1330 100644 --- a/src/oops/generic/instantiateObsErrorFactory.h +++ b/src/oops/generic/instantiateObsErrorFactory.h @@ -11,7 +11,7 @@ #ifndef OOPS_GENERIC_INSTANTIATEOBSERRORFACTORY_H_ #define OOPS_GENERIC_INSTANTIATEOBSERRORFACTORY_H_ -#include "oops/base/ObsErrorBase.h" +#include "oops/generic/ObsErrorBase.h" #include "oops/generic/ObsErrorDiag.h" namespace oops { diff --git a/src/oops/generic/instantiateTlmFactory.h b/src/oops/generic/instantiateTlmFactory.h deleted file mode 100644 index aa55fc874..000000000 --- a/src/oops/generic/instantiateTlmFactory.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_GENERIC_INSTANTIATETLMFACTORY_H_ -#define OOPS_GENERIC_INSTANTIATETLMFACTORY_H_ - -#include "oops/base/LinearModelBase.h" -#include "oops/generic/LinearModelId.h" - -namespace oops { - -template void instantiateTlmFactory() { - static LinearModelMaker > makerTLMID_("identity"); -} - -} // namespace oops - -#endif // OOPS_GENERIC_INSTANTIATETLMFACTORY_H_ diff --git a/src/oops/generic/unstructured_interpolation_mod.F90 b/src/oops/generic/unstructured_interpolation_mod.F90 index a44225d11..36725cb7a 100644 --- a/src/oops/generic/unstructured_interpolation_mod.F90 +++ b/src/oops/generic/unstructured_interpolation_mod.F90 @@ -175,7 +175,7 @@ subroutine create_new( self, comm, nn, wtype, ngrid_in, lats_in, lons_in, & index2 = self%interp_i(kk,n) dist = ageometry%distance(lons_in_glo(index1),lats_in_glo(index1),& lons_in_glo(index2),lats_in_glo(index2)) - wprod = wprod * dist + wprod = wprod * max(dist, 1e-10) endif enddo bw(jj) = 1.0_kind_real / wprod diff --git a/src/oops/interface/ChangeVariables.h b/src/oops/interface/ChangeVariables.h index 33d9f759f..791fc2380 100644 --- a/src/oops/interface/ChangeVariables.h +++ b/src/oops/interface/ChangeVariables.h @@ -12,9 +12,9 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/interface/VariableChange.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" diff --git a/src/oops/interface/ErrorCovariance.h b/src/oops/interface/ErrorCovariance.h index 9704694ee..9ba2cf442 100644 --- a/src/oops/interface/ErrorCovariance.h +++ b/src/oops/interface/ErrorCovariance.h @@ -18,11 +18,11 @@ #include "eckit/system/ResourceUsage.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" diff --git a/src/oops/interface/GeoVaLs.h b/src/oops/interface/GeoVaLs.h index b63e5690f..33f6c0b15 100644 --- a/src/oops/interface/GeoVaLs.h +++ b/src/oops/interface/GeoVaLs.h @@ -13,6 +13,7 @@ #include #include +#include #include @@ -37,7 +38,10 @@ class GeoVaLs : public util::Printable, public: static const std::string classname() {return "oops::GeoVaLs";} - GeoVaLs(const Locations_ &, const Variables &); + /// Allocate GeoVaLs for \p locs locations, to be filled with \p vars variables. + /// Sizes of GeoVaLs for i-th variable at a single location are defined by + /// i-th value of \p sizes. + GeoVaLs(const Locations_ & locs, const Variables &, const std::vector & sizes); GeoVaLs(const eckit::Configuration &, const ObsSpace_ &, const Variables &); GeoVaLs(const GeoVaLs &); @@ -69,10 +73,11 @@ class GeoVaLs : public util::Printable, // ----------------------------------------------------------------------------- template -GeoVaLs::GeoVaLs(const Locations_ & locs, const Variables & vars) : gvals_() { +GeoVaLs::GeoVaLs(const Locations_ & locs, const Variables & vars, + const std::vector & sizes) : gvals_() { Log::trace() << "GeoVaLs::GeoVaLs starting" << std::endl; util::Timer timer(classname(), "GeoVaLs"); - gvals_.reset(new GeoVaLs_(locs.locations(), vars)); + gvals_.reset(new GeoVaLs_(locs.locations(), vars, sizes)); Log::trace() << "GeoVaLs::GeoVaLs done" << std::endl; } diff --git a/src/oops/interface/Geometry.h b/src/oops/interface/Geometry.h index 424f1b092..29798fb02 100644 --- a/src/oops/interface/Geometry.h +++ b/src/oops/interface/Geometry.h @@ -19,6 +19,7 @@ #include "atlas/field.h" #include "atlas/functionspace.h" +#include "oops/base/Variables.h" #include "oops/interface/GeometryIterator.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" @@ -83,11 +84,21 @@ class Geometry : public util::Printable, /// Values of vertical coordinate in units specified by string (only used in GETKF) std::vector verticalCoord(std::string &) const; + /// Returns number of values required to store each of the passed model variables + /// \p vars at a single location (e.g. number of vertical levels). + /// The returned vector should be the same size as \p vars and contain number of + /// levels required for i-th variable in the i-th position. + std::vector variableSizes(const Variables &) const; + /// Accessor to the geometry communicator const eckit::mpi::Comm & getComm() const {return geom_->getComm();} atlas::FunctionSpace * atlasFunctionSpace() const {return geom_->atlasFunctionSpace();} atlas::FieldSet * atlasFieldSet() const {return geom_->atlasFieldSet();} + /// Accessor to MODEL::Geometry, used in the other interface classes in oops. + /// Does not need to be implemented. + const Geometry_ & geometry() const {return *this->geom_;} + protected: std::shared_ptr geom_; /// pointer to the Geometry implementation @@ -157,6 +168,17 @@ std::vector Geometry::verticalCoord(std::string & str) const { // ----------------------------------------------------------------------------- +template +std::vector Geometry::variableSizes(const Variables & vars) const { + Log::trace() << "Geometry::variableSizes starting" << std::endl; + util::Timer timer(classname(), "variableSizes"); + std::vector sizes = geom_->variableSizes(vars); + Log::trace() << "Geometry::variableSizes done" << std::endl; + return sizes; +} + +// ----------------------------------------------------------------------------- + template GeometryIterator Geometry::end() const { Log::trace() << "Geometry::end starting" << std::endl; @@ -177,64 +199,6 @@ void Geometry::print(std::ostream & os) const { } // namespace interface -// ----------------------------------------------------------------------------- -/// \brief Geometry class used in oops; subclass of interface class above -/// -/// \details Handles additional MPI communicator parameter in the constructors -/// (for MPI distribution in time, used in oops for 4DEnVar and weak-constraint 4DVar). -/// Adds extra methods that do not need to be implemented in the implementations: -/// - geometry() (for interfacing in oops) -/// - timeComm() (accessor to the MPI communicator in time) -template -class Geometry : public interface::Geometry { - typedef typename MODEL::Geometry Geometry_; - public: - typedef typename interface::Geometry::Parameters_ Parameters_; - - /// Constructor from Parameters and mpi communicators: \p geometry for spatial distribution - /// (handled by the implementation) and \p time for distribution in time (handled by oops) - Geometry(const Parameters_ &, const eckit::mpi::Comm & geometry, - const eckit::mpi::Comm & time); - /// Constructor from Configuration and mpi communicators: \p geometry for spatial distribution - /// (handled by the implementation) and \p time for distribution in time (handled by oops) - Geometry(const eckit::Configuration &, const eckit::mpi::Comm & geometry, - const eckit::mpi::Comm & time = oops::mpi::myself()); - /// Constructor from pointer to the MODEL::Geometry (used in 1DVar filter) - explicit Geometry(std::shared_ptr); - - /// Interfacing with other oops classes - const Geometry_ & geometry() const {return *this->geom_;} - - /// Accessor to the MPI communicator for distribution in time - const eckit::mpi::Comm & timeComm() const {return *timeComm_;} - - private: - const eckit::mpi::Comm * timeComm_; /// pointer to the MPI communicator in time -}; - -// ----------------------------------------------------------------------------- - -template -Geometry::Geometry(const eckit::Configuration & config, - const eckit::mpi::Comm & geometry, const eckit::mpi::Comm & time): - interface::Geometry(config, geometry), timeComm_(&time) -{} - -// ----------------------------------------------------------------------------- - -template -Geometry::Geometry(const Parameters_ & parameters, - const eckit::mpi::Comm & geometry, const eckit::mpi::Comm & time): - interface::Geometry(parameters, geometry), timeComm_(&time) -{} - -// ----------------------------------------------------------------------------- - -template -Geometry::Geometry(std::shared_ptr ptr): - interface::Geometry(ptr), timeComm_(&oops::mpi::myself()) -{} - } // namespace oops #endif // OOPS_INTERFACE_GEOMETRY_H_ diff --git a/src/oops/interface/GetValues.h b/src/oops/interface/GetValues.h index 9b8ca2b41..0af5e5db6 100644 --- a/src/oops/interface/GetValues.h +++ b/src/oops/interface/GetValues.h @@ -14,10 +14,10 @@ #include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index c00138cac..57f64107c 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. - * (C) Copyright 2017-2020 UCAR. + * (C) Copyright 2017-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -19,15 +19,13 @@ #include "atlas/field.h" #include "oops/base/GeneralizedDepartures.h" +#include "oops/base/Geometry.h" #include "oops/base/LocalIncrement.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/State.h" -#include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" -#include "oops/util/gatherPrint.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Serializable.h" #include "oops/util/Timer.h" @@ -45,7 +43,7 @@ class Increment : public oops::GeneralizedDepartures, typedef typename MODEL::Increment Increment_; typedef oops::Geometry Geometry_; typedef GeometryIterator GeometryIterator_; - typedef State State_; + typedef oops::State State_; public: static const std::string classname() {return "oops::Increment";} @@ -112,7 +110,17 @@ class Increment : public oops::GeneralizedDepartures, /// Accessor to geometry associated with this Increment Geometry_ geometry() const; - /// ATLAS FieldSet (used in SABER) + /// ATLAS FieldSet interface (used to communicate data with SABER) + /// For models that are not using ATLAS fields for their own Increment data: + /// - "setAtlas" allocates the ATLAS fields based on the variables present in the Increment. + /// - "toAtlas" allocates the ATLAS fields if necessary and copies Increment data into ATLAS + /// fields. + /// - "fromAtlas" copies ATLAS fields data into the Increment. + /// For models that are using ATLAS fields for their own Incerment data + /// - "setAtlas" copies ATLAS fields pointers from the Increment to the ATLAS fieldset. + /// - "toAtlas" copies data if ATLAS fields pointers in the Increment and the ATLAS fieldset are + /// different, and does nothing if pointers are the same. + /// - "fromAtlas" does nothing. void setAtlas(atlas::FieldSet *) const; void toAtlas(atlas::FieldSet *) const; void fromAtlas(atlas::FieldSet *); @@ -122,7 +130,8 @@ class Increment : public oops::GeneralizedDepartures, void serialize(std::vector &) const override; void deserialize(const std::vector &, size_t &) override; - /// Interfacing with other oops classes (returns reference to the implementation object) + /// Accessor to MODEL::Increment, used in the other interface classes in oops. + /// Does not need to be implemented. const Increment_ & increment() const {return *this->increment_;} Increment_ & increment() {return *this->increment_;} @@ -452,7 +461,6 @@ void Increment::deserialize(const std::vector & vect, size_t & cu // ----------------------------------------------------------------------------- - template void Increment::print(std::ostream & os) const { Log::trace() << "Increment::print starting" << std::endl; @@ -461,193 +469,9 @@ void Increment::print(std::ostream & os) const { Log::trace() << "Increment::print done" << std::endl; } -} // namespace interface - -// ----------------------------------------------------------------------------- -/// \brief Increment class used in oops -/// -/// \details -/// Adds extra methods that do not need to be implemented in the implementations: -/// - timeComm() (accessor to the MPI communicator in time - collection of processes -/// holding the data needed to represent the state in a particular region -/// of space X_i and throughout the whole time interval for which DA is done) -/// - variables() (accessor to variables in this Increment) -/// - shift_forward -/// - shift_backward -/// - toAtlas, atlas -/// -/// Adds communication through time to the following Increment methods: -/// - dot_product_with -/// - norm -/// - print - -template -class Increment : public interface::Increment { - typedef Geometry Geometry_; - - public: - /// Constructor for specified \p geometry, with \p variables, valid on \p date - Increment(const Geometry_ & geometry, const Variables & variables, const util::DateTime & date); - /// Copies \p other increment, changing its resolution to \p geometry - Increment(const Geometry_ & geometry, const Increment & other); - /// Creates Increment with the same geometry and variables as \p other. - /// Copies \p other if \p copy is true, otherwise creates zero increment - Increment(const Increment & other, const bool copy = true); - - /// Accessor to the time communicator - const eckit::mpi::Comm & timeComm() const {return *timeComm_;} - /// Accessor to Variables stored in this increment - const Variables & variables() const {return variables_;} - - /// Shift forward in time by \p dt - void shift_forward(const util::DateTime & dt); - /// Shift backward in time by \p dt - void shift_backward(const util::DateTime & dt); - - /// Set ATLAS fieldset associated with this Increment internally - void toAtlas(); - /// Allow to access base class's method as well - using interface::Increment::toAtlas; - /// Accessors to the ATLAS fieldset - atlas::FieldSet & atlas() {return atlasFieldSet_;} - const atlas::FieldSet & atlas() const {return atlasFieldSet_;} - - /// dot product with the \p other increment - double dot_product_with(const Increment & other) const; - /// Norm for diagnostics - double norm() const; - - private: - void print(std::ostream &) const override; - - Variables variables_; /// Variables stored in this Increment - const eckit::mpi::Comm * timeComm_; /// pointer to the MPI communicator in time - atlas::FieldSet atlasFieldSet_; /// Atlas fields associated with this Increment -}; - -// ----------------------------------------------------------------------------- - -template -Increment::Increment(const Geometry_ & geometry, const Variables & variables, - const util::DateTime & date): - interface::Increment(geometry, variables, date), variables_(variables), - timeComm_(&geometry.timeComm()) -{} - -// ----------------------------------------------------------------------------- - -template -Increment::Increment(const Geometry_ & geometry, const Increment & other): - interface::Increment(geometry, other), variables_(other.variables_), - timeComm_(other.timeComm_) -{} - -// ----------------------------------------------------------------------------- - -template -Increment::Increment(const Increment & other, const bool copy): - interface::Increment(other, copy), variables_(other.variables_), - timeComm_(other.timeComm_) -{} - -// ----------------------------------------------------------------------------- - -template -double Increment::dot_product_with(const Increment & dx) const { - double zz = interface::Increment::dot_product_with(dx); - timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); - return zz; -} - -// ----------------------------------------------------------------------------- - -template -double Increment::norm() const { - double zz = interface::Increment::norm(); - zz *= zz; - timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); - zz = sqrt(zz); - return zz; -} - -// ----------------------------------------------------------------------------- - -template -void Increment::shift_forward(const util::DateTime & begin) { - Log::trace() << "Increment::Increment shift_forward starting" << std::endl; - static int tag = 159357; - size_t mytime = timeComm_->rank(); - -// Send values of M.dx_i at end of my subwindow to next subwindow - if (mytime + 1 < timeComm_->size()) { - oops::mpi::send(*timeComm_, *this, mytime+1, tag); - } - -// Receive values at beginning of my subwindow from previous subwindow - if (mytime > 0) { - oops::mpi::receive(*timeComm_, *this, mytime-1, tag); - } else { - this->zero(begin); - } - - ++tag; - Log::trace() << "Increment::Increment shift_forward done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Increment::shift_backward(const util::DateTime & end) { - Log::trace() << "Increment::Increment shift_backward starting" << std::endl; - static int tag = 30951; - size_t mytime = timeComm_->rank(); - -// Send values of dx_i at start of my subwindow to previous subwindow - if (mytime > 0) { - oops::mpi::send(*timeComm_, *this, mytime-1, tag); - } - -// Receive values at end of my subwindow from next subwindow - if (mytime + 1 < timeComm_->size()) { - oops::mpi::receive(*timeComm_, *this, mytime+1, tag); - } else { - this->zero(end); - } - - ++tag; - Log::trace() << "Increment::Increment shift_backward done" << std::endl; -} - - -// ----------------------------------------------------------------------------- -template -void Increment::toAtlas() { - interface::Increment::toAtlas(&atlasFieldSet_); - this->increment_.reset(); -} - // ----------------------------------------------------------------------------- -template -void Increment::print(std::ostream & os) const { - if (timeComm_->size() > 1) { - gatherPrint(os, this->increment(), *timeComm_); - } else { - os << this->increment(); - } -} - -// ----------------------------------------------------------------------------- -/// Add on \p dx incrment to model state \p xx -template -State & operator+=(State & xx, const Increment & dx) { - Log::trace() << "operator+=(State, Increment) starting" << std::endl; - util::Timer timer("oops::Increment", "operator+=(State, Increment)"); - xx.state() += dx.increment(); - Log::trace() << "operator+=(State, Increment) done" << std::endl; - return xx; -} - +} // namespace interface } // namespace oops diff --git a/src/oops/interface/LinearGetValues.h b/src/oops/interface/LinearGetValues.h index df530b03e..6ba8ce386 100644 --- a/src/oops/interface/LinearGetValues.h +++ b/src/oops/interface/LinearGetValues.h @@ -14,11 +14,11 @@ #include "eckit/config/Configuration.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/interface/GeoVaLs.h" -#include "oops/interface/Increment.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/ObjectCounter.h" diff --git a/src/oops/interface/LinearModelBase.h b/src/oops/interface/LinearModelBase.h new file mode 100644 index 000000000..1ef7d1c68 --- /dev/null +++ b/src/oops/interface/LinearModelBase.h @@ -0,0 +1,140 @@ +/* + * (C) Copyright 2018-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_LINEARMODELBASE_H_ +#define OOPS_INTERFACE_LINEARMODELBASE_H_ + +#include +#include + +#include + +#include "oops/generic/LinearModelBase.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/Increment.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/interface/ModelAuxIncrement.h" +#include "oops/interface/State.h" +#include "oops/util/Logger.h" + +namespace oops { + +namespace interface { + +// ----------------------------------------------------------------------------- + +/// \brief Base class for MODEL-specific implementations of the LinearModel interface. +/// interface::LinearModelBase overrides oops::LinearModelBase methods to pass MODEL-specific +/// implementations of State, Increment and ModelAuxIncrement to the MODEL-specific implementation +/// of LinearModel. +/// +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of LinearModelParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// LinearModelBase(const Geometry_ &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of LinearModelParametersBase +/// holding the settings of the linear model in question. The implementation of the LinearModelBase +/// interface should then typedef `Parameters_` to the name of that subclass and provide a +/// constructor with the following signature: +/// +/// LinearModelBase(const Geometry_ &, const Parameters_ &); +/// +template +class LinearModelBase : public oops::LinearModelBase { + typedef typename MODEL::Increment Increment_; + typedef typename MODEL::ModelAuxControl ModelAuxCtl_; + typedef typename MODEL::ModelAuxIncrement ModelAuxInc_; + typedef typename MODEL::State State_; + + public: + static const std::string classname() {return "oops::interface::LinearModelBase";} + + LinearModelBase() = default; + virtual ~LinearModelBase() = default; + + /// Overrides for oops::LinearModelBase classes, passing MODEL-specific classes to the + /// MODEL-specific implementations of LinearModel + void initializeTL(oops::Increment & dx) const final + { this->initializeTL(dx.increment()); } + void stepTL(oops::Increment & dx, const ModelAuxIncrement & modelaux) const final + { this->stepTL(dx.increment(), modelaux.modelauxincrement()); } + void finalizeTL(oops::Increment & dx) const final + { this->finalizeTL(dx.increment()); } + + void initializeAD(oops::Increment & dx) const final + { this->initializeAD(dx.increment()); } + void stepAD(oops::Increment & dx, ModelAuxIncrement & modelaux) const final + { this->stepAD(dx.increment(), modelaux.modelauxincrement()); } + void finalizeAD(oops::Increment & dx) const final + { this->finalizeAD(dx.increment()); } + + void setTrajectory(const oops::State & xx, oops::State & xxtraj, + const ModelAuxControl & modelaux) final + { this->setTrajectory(xx.state(), xxtraj.state(), modelaux.modelauxcontrol()); } + + /// \brief Tangent linear forecast initialization, called before every run + virtual void initializeTL(Increment_ &) const = 0; + /// \brief Tangent linear forecast "step", called during run; updates Increment to the next time + virtual void stepTL(Increment_ &, const ModelAuxInc_ &) const = 0; + /// \brief Tangent linear forecast finalization; called after each run + virtual void finalizeTL(Increment_ &) const = 0; + + /// \brief Adjoint forecast initialization, called before every run + virtual void initializeAD(Increment_ &) const = 0; + /// \brief Adjoint forecast "step", called during run; updates increment to the previous time + virtual void stepAD(Increment_ &, ModelAuxInc_ &) const = 0; + /// \brief Adjoint forecast finalization; called after each run + virtual void finalizeAD(Increment_ &) const = 0; + + /// \brief Set the trajectory for the linear model, called after each step of the forecast. + /// The incoming State is output from the nonlinear forecast. The adjustable State is + /// interpolated to the resolution of the linear model. + virtual void setTrajectory(const State_ &, State_ &, const ModelAuxCtl_ &) = 0; + + /// \brief Print, used in logging + void print(std::ostream &) const = 0; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of LinearModelFactory able to create instances of T (a concrete subclass of +/// interface::LinearModelBase). Passes MODEL::Geometry to the constructor of T. +template +class LinearModelMaker : public LinearModelFactory { + private: + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericLinearModelParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + public: + typedef oops::Geometry Geometry_; + + explicit LinearModelMaker(const std::string & name) : LinearModelFactory(name) {} + + oops::LinearModelBase * make(const Geometry_ & geom, + const LinearModelParametersBase & parameters) override { + Log::trace() << "interface::LinearModelBase::make starting" << std::endl; + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return new T(geom.geometry(), + parametersOrConfiguration::value>(stronglyTypedParameters)); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } +}; + +// ----------------------------------------------------------------------------- + +} // namespace interface + +} // namespace oops + +#endif // OOPS_INTERFACE_LINEARMODELBASE_H_ diff --git a/src/oops/interface/LinearObsOperator.h b/src/oops/interface/LinearObsOperator.h index 887ed3c3d..89668fb68 100644 --- a/src/oops/interface/LinearObsOperator.h +++ b/src/oops/interface/LinearObsOperator.h @@ -16,13 +16,12 @@ #include +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsAuxControl.h" #include "oops/interface/ObsAuxIncrement.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/util/DateTime.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -31,7 +30,14 @@ namespace oops { // ----------------------------------------------------------------------------- - +/// \brief MODEL-agnostic part of tangent-linear and adjoint of the nonlinear +/// observation (forward) operator ObsOperator. +/// +/// Note: each implementation should typedef `Parameters_` to the name of a subclass of +/// oops::Parameters holding its configuration settings and provide a constructor with the +/// following signature: +/// +/// LinearObsOperator(const OBS::ObsSpace &, const Parameters_ &); template class LinearObsOperator : public util::Printable, private boost::noncopyable, @@ -44,24 +50,55 @@ class LinearObsOperator : public util::Printable, typedef ObsVector ObsVector_; public: + /// A subclass of oops::Parameters holding the configuration settings of the operator. + typedef typename LinearObsOper_::Parameters_ Parameters_; + static const std::string classname() {return "oops::LinearObsOperator";} - LinearObsOperator(const ObsSpace_ &, const eckit::Configuration &); + /// Set up TL and AD of observation operator for the \p obsspace observations, with + /// parameters defined in \p parameters. + LinearObsOperator(const ObsSpace_ & obsspace, const Parameters_ & parameters); ~LinearObsOperator(); -/// Interfacing - const LinearObsOper_ & linearobsoperator() const {return *oper_;} - -/// Obs Operators - void setTrajectory(const GeoVaLs_ &, const ObsAuxControl_ &); - void simulateObsTL(const GeoVaLs_ &, ObsVector_ &, const ObsAuxIncrement_ &) const; - void simulateObsAD(GeoVaLs_ &, const ObsVector_ &, ObsAuxIncrement_ &) const; - -/// Other - const Variables & requiredVars() const; // Required inputs variables from LinearModel + /// Sets up the trajectory for future calls of simulateObsTL or simulateObsAD. + /// The implementations could e.g. save the trajectory \p x0, or compute and save the Jacobian + /// of observation operator around \p x0. + /// Always called before simulateObsTL or simulateObsAD. + /// \param[in] x0 trajectory for linearization of obs operator, State interpolated + /// to observations locations (defined by ObsOperator::locations()) + /// \param[in] obsaux additional obs operator input, used in the minimization + /// in Variational DA, e.g. bias correction coefficients or obs operator + /// parameters. + void setTrajectory(const GeoVaLs_ & x0, const ObsAuxControl_ & obsaux); + + /// Apply tangent-linear of the observation operator linearized around the trajectory that was + /// passed to setTrajectory method (which is always called before simulateObsTL). + /// \param[in] dx input to the TL obs operator, Increment interpolated to observations + /// locations. + /// \param[out] dy output of the TL obs operator. + /// \param[in] dobsaux: additional input to the TL obs operator, e.g. perturbation to bias + /// coefficients or obs operator parameters. + void simulateObsTL(const GeoVaLs_ & dx, ObsVector_ & dy, const ObsAuxIncrement_ & dobsaux) const; + /// Apply adjoint of the observation operator linearized around the trajectory that was + /// passed to setTrajectory method (which is always called before simulateObsAD). + /// \param[out] dx output of the AD obs operator, Increment interpolated to observations + /// locations. + /// \param[in] dy input of the AD obs operator, perturbation to the ObsVector. + /// \param[out] dobsaux additional output of the AD obs operator, e.g. perturbation to bias + /// coefficients or obs operator parameters. + void simulateObsAD(GeoVaLs_ & dx, const ObsVector_ & dy, ObsAuxIncrement_ & dobsaux) const; + + /// Variables required from the model Increment to compute TL or AD of the obs operator. + /// These variables will be provided in GeoVaLs passed to simulateObsTL and simulateObsAD. + /// Note: these Variables may be different from variables returned by ObsOperator::requiredVars(), + /// which will be provided in GeoVaLs passed to setTrajectory. + const Variables & requiredVars() const; private: + /// Print, used for logging void print(std::ostream &) const; + + /// Pointer to the implementation of LinearObsOperator std::unique_ptr oper_; }; @@ -69,10 +106,10 @@ class LinearObsOperator : public util::Printable, template LinearObsOperator::LinearObsOperator(const ObsSpace_ & os, - const eckit::Configuration & config): oper_() { + const Parameters_ & parameters): oper_() { Log::trace() << "LinearObsOperator::LinearObsOperator starting" << std::endl; util::Timer timer(classname(), "LinearObsOperator"); - oper_.reset(new LinearObsOper_(os.obsspace(), config)); + oper_.reset(new LinearObsOper_(os.obsspace(), parameters)); Log::trace() << "LinearObsOperator::LinearObsOperator done" << std::endl; } diff --git a/src/oops/interface/LinearVariableChange.h b/src/oops/interface/LinearVariableChange.h index 6c697162d..ea5e74403 100644 --- a/src/oops/interface/LinearVariableChange.h +++ b/src/oops/interface/LinearVariableChange.h @@ -13,11 +13,11 @@ #include +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" diff --git a/src/oops/interface/Localization.h b/src/oops/interface/Localization.h deleted file mode 100644 index 41827bc50..000000000 --- a/src/oops/interface/Localization.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -* Copyright 2011 ECMWF -* Copyright 2020-2020 UCAR -* -* This software was developed at ECMWF for evaluation -* and may be used for academic and research purposes only. -* The software is provided as is without any warranty. -* -* This software can be used, copied and modified but not -* redistributed or sold. This notice must be reproduced -* on each copy made. -*/ - -#ifndef OOPS_INTERFACE_LOCALIZATION_H_ -#define OOPS_INTERFACE_LOCALIZATION_H_ - -#include -#include - -#include "eckit/config/Configuration.h" -#include "oops/base/LocalizationBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" - -namespace oops { - -// ----------------------------------------------------------------------------- -/// \brief Model-space localization class: intended for model-specific implementations -template -class Localization : public LocalizationBase { - typedef Geometry Geometry_; - typedef Increment Increment_; - public: - static const std::string classname() {return "oops::Localization";} - - Localization(const Geometry_ &, const util::DateTime &, const eckit::Configuration &); - ~Localization(); - - void doRandomize(Increment_ &) const override; - void doMultiply(Increment_ &) const override; - - private: - void print(std::ostream &) const override; - - std::unique_ptr loc_; -}; - -// ----------------------------------------------------------------------------- - -template -Localization::Localization(const Geometry_ & geometry, - const util::DateTime & time, - const eckit::Configuration & conf) { - Log::trace() << "Localization::Localization starting" << std::endl; - util::Timer timer(classname(), "Localization"); - loc_.reset(new LOC(geometry.geometry(), conf)); - Log::trace() << "Localization::Localization done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -Localization::~Localization() { - Log::trace() << "Localization::~Localization starting" << std::endl; - util::Timer timer(classname(), "~Localization"); - loc_.reset(); - Log::trace() << "Localization::~Localization done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Localization::doRandomize(Increment_ & dx) const { - Log::trace() << "Localization::doRandomize starting" << std::endl; - util::Timer timer(classname(), "doRandomize"); - loc_->randomize(dx.increment()); - Log::trace() << "Localization::doRandomize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Localization::doMultiply(Increment_ & dx) const { - Log::trace() << "Localization::doMultiply starting" << std::endl; - util::Timer timer(classname(), "doMultiply"); - loc_->multiply(dx.increment()); - Log::trace() << "Localization::doMultiply done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Localization::print(std::ostream & os) const { - Log::trace() << "Localization::print starting" << std::endl; - util::Timer timer(classname(), "print"); - os << *loc_; - Log::trace() << "Localization::print done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_INTERFACE_LOCALIZATION_H_ diff --git a/src/oops/interface/LocalizationBase.h b/src/oops/interface/LocalizationBase.h new file mode 100644 index 000000000..491bff40a --- /dev/null +++ b/src/oops/interface/LocalizationBase.h @@ -0,0 +1,84 @@ +/* +* Copyright 2011 ECMWF +* Copyright 2020-2021 UCAR +* +* This software was developed at ECMWF for evaluation +* and may be used for academic and research purposes only. +* The software is provided as is without any warranty. +* +* This software can be used, copied and modified but not +* redistributed or sold. This notice must be reproduced +* on each copy made. +*/ + +#ifndef OOPS_INTERFACE_LOCALIZATIONBASE_H_ +#define OOPS_INTERFACE_LOCALIZATIONBASE_H_ + +#include +#include + +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/generic/LocalizationBase.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +namespace interface { + +// ----------------------------------------------------------------------------- +/// Base class for MODEL-specific implementations of the Loclaization interface. +/// interface::LocalizationBase overrides oops::LocalizationBase methods to pass +/// MODEL-specific implementation of Increment to the MODEL-specific +/// implementation of Localization. +/// +/// Note: subclasses need to provide a constructor with the following signature: +/// +/// LocalizationBase(const MODEL::Geometry &, const eckit::Configuration &); +/// +template +class LocalizationBase : public oops::LocalizationBase { + typedef typename MODEL::Increment Increment_; + public: + static const std::string classname() {return "oops::Localization";} + + LocalizationBase() = default; + virtual ~LocalizationBase() = default; + + /// Overrides for oops::LocalizationBase classes, passing MODEL-specific classes to the + /// MODEL-specific implementations of Localization + void randomize(oops::Increment & dx) const final + { this->randomize(dx.increment()); } + void multiply(oops::Increment & dx) const final + { this->multiply(dx.increment()); } + + /// Randomize \p dx and apply 3D localization + virtual void randomize(Increment_ & dx) const = 0; + /// Apply 3D localization to \p dx + virtual void multiply(Increment_ & dx) const = 0; +}; + +// ----------------------------------------------------------------------------- +/// \brief A subclass of LocalizationFactory able to create instances of T (a concrete +/// subclass of interface::LocalizationBase). Passes MODEL::Geometry to the constructor of T. +template +class LocalizationMaker : public oops::LocalizationFactory { + typedef oops::Geometry Geometry_; + std::unique_ptr> make(const Geometry_ & geometry, + const eckit::Configuration & conf) override + { return std::make_unique(geometry.geometry(), conf); } + public: + explicit LocalizationMaker(const std::string & name) : oops::LocalizationFactory(name) {} +}; + + +// ----------------------------------------------------------------------------- + +} // namespace interface + +} // namespace oops + +#endif // OOPS_INTERFACE_LOCALIZATIONBASE_H_ diff --git a/src/oops/interface/ModelAuxControl.h b/src/oops/interface/ModelAuxControl.h index 1f372f04f..f6e951292 100644 --- a/src/oops/interface/ModelAuxControl.h +++ b/src/oops/interface/ModelAuxControl.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -15,8 +15,7 @@ #include #include - -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -28,6 +27,15 @@ namespace eckit { namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Auxiliary state related to model (could be e.g. model bias), not used at the moment. +/// \details +/// This class is used to manipulate parameters of the model that can be estimated in the +/// assimilation. This includes model bias but could be used for other parameters. +/// This is sometimes referred to as augmented state or augmented control variable in the +/// literature. +/// This class calls the model's implementation of ModelAuxControl. + // ----------------------------------------------------------------------------- template @@ -39,18 +47,26 @@ class ModelAuxControl : public util::Printable, public: static const std::string classname() {return "oops::ModelAuxControl";} - ModelAuxControl(const Geometry_ &, const eckit::Configuration &); - ModelAuxControl(const Geometry_ &, const ModelAuxControl &); + /// Constructor for specified \p resol and \p conf + ModelAuxControl(const Geometry_ & resol, const eckit::Configuration & conf); + /// Copies \p other ModelAuxControl, changing its resolution to \p resol + ModelAuxControl(const Geometry_ & resol, const ModelAuxControl & other); + /// Creates ModelAuxControl with the same structure as \p other. + /// Copies \p other if \p copy is true, otherwise creates zero ModelAuxControl explicit ModelAuxControl(const ModelAuxControl &, const bool copy = true); + /// Destructor (defined explicitly for timing and tracing) ~ModelAuxControl(); -/// Interfacing + /// const Accessor const ModelAuxControl_ & modelauxcontrol() const {return *aux_;} + /// Accessor ModelAuxControl_ & modelauxcontrol() {return *aux_;} -/// I/O and diagnostics + /// Read this ModelAuxControl from file void read(const eckit::Configuration &); + /// Write this ModelAuxControl out to file void write(const eckit::Configuration &) const; + /// Norm (used in tests) double norm() const; private: diff --git a/src/oops/interface/ModelAuxCovariance.h b/src/oops/interface/ModelAuxCovariance.h index c99e08473..076622806 100644 --- a/src/oops/interface/ModelAuxCovariance.h +++ b/src/oops/interface/ModelAuxCovariance.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -18,7 +18,7 @@ #include #include "eckit/config/Configuration.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/interface/ModelAuxControl.h" #include "oops/interface/ModelAuxIncrement.h" #include "oops/util/Logger.h" @@ -28,6 +28,10 @@ namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Auxiliary Error Covariance related to model, not used at the moment. +/// \details +/// This class calls the model's implementation of ModelAuxCovariance. // ----------------------------------------------------------------------------- template @@ -35,22 +39,28 @@ class ModelAuxCovariance : public util::Printable, private boost::noncopyable, private util::ObjectCounter > { typedef typename MODEL::ModelAuxCovariance ModelAuxCovariance_; - typedef Geometry Geometry_; + typedef Geometry Geometry_; typedef ModelAuxControl ModelAuxControl_; - typedef ModelAuxIncrement ModelAuxIncrement_; + typedef ModelAuxIncrement ModelAuxIncrement_; public: static const std::string classname() {return "oops::ModelAuxCovariance";} - ModelAuxCovariance(const eckit::Configuration &, const Geometry_ &); + /// Constructor for specified \p conf and \p resol + ModelAuxCovariance(const eckit::Configuration & conf, const Geometry_ & resol); + /// Destructor (defined explicitly for timing and tracing) ~ModelAuxCovariance(); -/// Operators + /// linearize operator void linearize(const ModelAuxControl_ &, const Geometry_ &); + /// Sets the second parameter to the first multiplied by the covariance matrix. void multiply(const ModelAuxIncrement_ &, ModelAuxIncrement_ &) const; + /// Sets the second parameter to the first multiplied by the inverse covariance matrix. void inverseMultiply(const ModelAuxIncrement_ &, ModelAuxIncrement_ &) const; + /// randomize the values in the ModelAuxIncrement void randomize(ModelAuxIncrement_ &) const; + /// Accessor to the configuration associated with the ModelAuxIncrement const eckit::Configuration & config() const {return cov_->config();} private: diff --git a/src/oops/interface/ModelAuxIncrement.h b/src/oops/interface/ModelAuxIncrement.h index 6593d0b3f..af581a302 100644 --- a/src/oops/interface/ModelAuxIncrement.h +++ b/src/oops/interface/ModelAuxIncrement.h @@ -16,8 +16,7 @@ #include #include - -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/interface/ModelAuxControl.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" @@ -31,6 +30,10 @@ namespace eckit { namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Auxiliary Increment related to model, not used at the moment. +/// \details +/// This class calls the model's implementation of ModelAuxIncrement. // ----------------------------------------------------------------------------- template @@ -38,38 +41,48 @@ class ModelAuxIncrement : public util::Printable, public util::Serializable, private util::ObjectCounter > { typedef typename MODEL::ModelAuxIncrement ModelAuxIncrement_; - typedef Geometry Geometry_; + typedef Geometry Geometry_; typedef ModelAuxControl ModelAuxControl_; public: static const std::string classname() {return "oops::ModelAuxIncrement";} -/// Constructor, destructor - ModelAuxIncrement(const Geometry_ &, const eckit::Configuration &); - explicit ModelAuxIncrement(const ModelAuxIncrement &, const bool copy = true); - ModelAuxIncrement(const ModelAuxIncrement &, const eckit::Configuration &); + /// Constructor for specified \p resol and \p conf + ModelAuxIncrement(const Geometry_ & resol, const eckit::Configuration & conf); + /// Copies \p other ModelAuxIncrement if \p copy is true, + /// otherwise creates zero ModelAuxIncrement with same variables and geometry + explicit ModelAuxIncrement(const ModelAuxIncrement & other, const bool copy = true); + /// Copies \p other ModelAuxIncrement, reading extra information from \p conf + ModelAuxIncrement(const ModelAuxIncrement & other, const eckit::Configuration & conf); + /// Destructor (defined explicitly for timing and tracing) ~ModelAuxIncrement(); -/// Interfacing + /// const Accessor const ModelAuxIncrement_ & modelauxincrement() const {return *aux_;} + /// Accessor ModelAuxIncrement_ & modelauxincrement() {return *aux_;} -/// Linear algebra operators + /// Sets this ModelAuxIncrement to the difference between two ModelAuxControl objects void diff(const ModelAuxControl_ &, const ModelAuxControl_ &); + /// Zero out this ModelAuxIncrement void zero(); + /// Linear algebra operators ModelAuxIncrement & operator=(const ModelAuxIncrement &); ModelAuxIncrement & operator+=(const ModelAuxIncrement &); ModelAuxIncrement & operator-=(const ModelAuxIncrement &); ModelAuxIncrement & operator*=(const double &); void axpy(const double &, const ModelAuxIncrement &); - double dot_product_with(const ModelAuxIncrement &) const; + /// dot product with the \p other ModelAuxIncrement + double dot_product_with(const ModelAuxIncrement & other) const; -/// I/O and diagnostics + /// Read this ModelAuxIncrement from file void read(const eckit::Configuration &); + /// Write this ModelAuxIncrement out to file void write(const eckit::Configuration &) const; + /// Norm (used in tests) double norm() const; -/// Serialize and deserialize + /// Serialize and deserialize (used in 4DEnVar, weak-constraint 4DVar and Block-Lanczos minimizer) size_t serialSize() const override; void serialize(std::vector &) const override; void deserialize(const std::vector &, size_t &) override; diff --git a/src/oops/interface/ModelBase.h b/src/oops/interface/ModelBase.h new file mode 100644 index 000000000..b197fad41 --- /dev/null +++ b/src/oops/interface/ModelBase.h @@ -0,0 +1,110 @@ +/* + * (C) Copyright 2018-2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_MODELBASE_H_ +#define OOPS_INTERFACE_MODELBASE_H_ + +#include +#include + +#include + +#include "oops/base/State.h" +#include "oops/generic/ModelBase.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/util/Logger.h" + +namespace oops { + +namespace interface { + +// ----------------------------------------------------------------------------- + +/// \brief Base class for MODEL-specific implementations of the Model interface. +/// interface::ModelBase overrides oops::ModelBase methods to pass MODEL-specific +/// implementations of State and ModelAuxControl to the MODEL-specific +/// implementation of Model. +/// +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of ModelParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// ModelBase(const Geometry_ &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of ModelParametersBase +/// holding the settings of the model in question. The implementation of the ModelBase interface +/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with +/// the following signature: +/// +/// ModelBase(const Geometry_ &, const Parameters_ &); +/// +template +class ModelBase : public oops::ModelBase { + typedef typename MODEL::ModelAuxControl ModelAux_; + typedef typename MODEL::State State_; + + public: + static const std::string classname() {return "oops::interface::ModelBase";} + + ModelBase() = default; + virtual ~ModelBase() = default; + + /// Overrides for oops::ModelBase classes, passing MODEL-specific classes to the + /// MODEL-specific implementations of Model + void initialize(oops::State & xx) const final + { this->initialize(xx.state()); } + void step(oops::State & xx, const ModelAuxControl & modelaux) const final + { this->step(xx.state(), modelaux.modelauxcontrol()); } + void finalize(oops::State & xx) const final + { this->finalize(xx.state()); } + + /// \brief Forecast initialization, called before every forecast run + virtual void initialize(State_ &) const = 0; + /// \brief Forecast "step", called during forecast run; updates state to the next time + virtual void step(State_ &, const ModelAux_ &) const = 0; + /// \brief Forecast finalization; called after each forecast run + virtual void finalize(State_ &) const = 0; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of ModelFactory able to create instances of T (a concrete subclass of +/// interface::ModelBase). Passes MODEL::Geometry to the constructor of T. +template +class ModelMaker : public ModelFactory { + private: + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericModelParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + public: + typedef oops::Geometry Geometry_; + + explicit ModelMaker(const std::string & name) : ModelFactory(name) {} + + oops::ModelBase * make(const Geometry_ & geom, + const ModelParametersBase & parameters) override { + Log::trace() << "interface::ModelBase::make starting" << std::endl; + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return new T(geom.geometry(), + parametersOrConfiguration::value>(stronglyTypedParameters)); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } +}; + +// ----------------------------------------------------------------------------- + +} // namespace interface + +} // namespace oops + +#endif // OOPS_INTERFACE_MODELBASE_H_ diff --git a/src/oops/interface/ObsAuxControl.h b/src/oops/interface/ObsAuxControl.h index 84d0a038f..584fc18d4 100644 --- a/src/oops/interface/ObsAuxControl.h +++ b/src/oops/interface/ObsAuxControl.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -25,6 +25,11 @@ namespace oops { class Variables; // ----------------------------------------------------------------------------- +/// \brief Auxiliary state related to observations, templated on +/// \details +/// This is currently only used for bias correction coefficients, but can be used for other cases. +/// This class calls the implementation of ObsAuxControl. +// ----------------------------------------------------------------------------- template class ObsAuxControl : public util::Printable, @@ -36,25 +41,33 @@ class ObsAuxControl : public util::Printable, static const std::string classname() {return "oops::ObsAuxControl";} - ObsAuxControl(const ObsSpace &, const Parameters_ ¶ms); + /// Constructor for specified ObsSpace \p os and \p params + ObsAuxControl(const ObsSpace & os, const Parameters_ & params); + /// Creates ObsAuxControl with the same structure as \p other. + /// Copies \p other if \p copy is true, otherwise creates zero ObsAuxControl explicit ObsAuxControl(const ObsAuxControl &, const bool copy = true); + /// Destructor (defined explicitly for timing and tracing) ~ObsAuxControl(); -/// Interfacing + /// const Accessor const ObsAuxControl_ & obsauxcontrol() const {return *aux_;} + /// Accessor ObsAuxControl_ & obsauxcontrol() {return *aux_;} -/// I/O and diagnostics + /// Read this ObsAuxControl from file void read(const Parameters_ &); + /// Write this ObsAuxControl out to file void write(const Parameters_ &) const; + /// Norm (used in tests) double norm() const; -/// Other + /// Return required inputs variables from Model const Variables & requiredVars() const; + /// Return required observations diagnostics const Variables & requiredHdiagnostics() const; -/// Operator - ObsAuxControl & operator=(const ObsAuxControl &); + /// Assign operator from other ObsAuxControl \p rhs + ObsAuxControl & operator=(const ObsAuxControl & rhs); private: void print(std::ostream &) const; diff --git a/src/oops/interface/ObsAuxCovariance.h b/src/oops/interface/ObsAuxCovariance.h index 95d238804..f4c3f572d 100644 --- a/src/oops/interface/ObsAuxCovariance.h +++ b/src/oops/interface/ObsAuxCovariance.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -28,6 +28,11 @@ namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Auxiliary error covariance related to observations, templated on +/// \details +/// This is currently only used for bias correction coefficient error covariances. +/// This class calls the implementation of ObsAuxCovariance. // ----------------------------------------------------------------------------- template @@ -43,13 +48,18 @@ class ObsAuxCovariance : public util::Printable, static const std::string classname() {return "oops::ObsAuxCovariance";} - ObsAuxCovariance(const ObsSpace &, const Parameters_ &); + /// Constructor for specified ObsSpace \p os and \p params + ObsAuxCovariance(const ObsSpace & os, const Parameters_ & params); + /// Destructor (defined explicitly for timing and tracing) ~ObsAuxCovariance(); -/// Operators + /// linearize operator void linearize(const ObsAuxControl_ &, const eckit::Configuration &); + /// Sets the second parameter to the first multiplied by the covariance matrix. void multiply(const ObsAuxIncrement_ &, ObsAuxIncrement_ &) const; + /// Sets the second parameter to the first multiplied by the inverse covariance matrix. void inverseMultiply(const ObsAuxIncrement_ &, ObsAuxIncrement_ &) const; + /// randomize the values in the ObsAuxIncrement void randomize(ObsAuxIncrement_ &) const; private: diff --git a/src/oops/interface/ObsAuxIncrement.h b/src/oops/interface/ObsAuxIncrement.h index 3a7ba9207..944209bf9 100644 --- a/src/oops/interface/ObsAuxIncrement.h +++ b/src/oops/interface/ObsAuxIncrement.h @@ -30,6 +30,11 @@ namespace eckit { namespace oops { +// ----------------------------------------------------------------------------- +/// \brief Auxiliary increment related to observations, templated on +/// \details +/// This is currently only used for bias correction coefficient increments. +/// This class calls the implementation of ObsAuxIncrement. // ----------------------------------------------------------------------------- template @@ -44,33 +49,40 @@ class ObsAuxIncrement : public util::Printable, static const std::string classname() {return "oops::ObsAuxIncrement";} -/// Constructor, destructor - ObsAuxIncrement(const ObsSpace &, const Parameters_ &); - /// Copies \p other if \p copy is true, otherwise creates zero increment + /// Constructor for specified ObsSpace \p os and \p params + ObsAuxIncrement(const ObsSpace & os, const Parameters_ & params); + /// Copies \p other if \p copy is true, otherwise creates zero ObsAuxIncrement /// of the same size as \p other. ObsAuxIncrement(const ObsAuxIncrement & other, const bool copy = true); + /// Destructor (defined explicitly for timing and tracing) ~ObsAuxIncrement(); -/// Interfacing + /// const Accessor const ObsAuxIncrement_ & obsauxincrement() const {return *aux_;} + /// Accessor ObsAuxIncrement_ & obsauxincrement() {return *aux_;} -/// Linear algebra operators + /// Sets this ObsAuxIncrement to the difference between two ObsAuxControl objects void diff(const ObsAuxControl_ &, const ObsAuxControl_ &); + /// Zero out this ObsAuxIncrement void zero(); + /// Linear algebra operators ObsAuxIncrement & operator=(const ObsAuxIncrement &); ObsAuxIncrement & operator+=(const ObsAuxIncrement &); ObsAuxIncrement & operator-=(const ObsAuxIncrement &); ObsAuxIncrement & operator*=(const double &); void axpy(const double &, const ObsAuxIncrement &); - double dot_product_with(const ObsAuxIncrement &) const; + /// dot product with \p dx ObsAuxIncrement + double dot_product_with(const ObsAuxIncrement & dx) const; -/// I/O and diagnostics + /// Read this ObsAuxIncrement from file void read(const eckit::Configuration &); + /// Write this ObsAuxIncrement out to file void write(const eckit::Configuration &) const; + /// Norm (used in tests) double norm() const; -/// Serialize and deserialize + /// Serialize and deserialize (used in 4DEnVar, weak-constraint 4DVar and Block-Lanczos minimizer) size_t serialSize() const override; void serialize(std::vector &) const override; void deserialize(const std::vector &, size_t &) override; diff --git a/src/oops/interface/ObsDataVector.h b/src/oops/interface/ObsDataVector.h index 90581213d..88d9ee95e 100644 --- a/src/oops/interface/ObsDataVector.h +++ b/src/oops/interface/ObsDataVector.h @@ -12,9 +12,9 @@ #include #include +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" diff --git a/src/oops/interface/ObsDataVector_head.h b/src/oops/interface/ObsDataVector_head.h index 5f2ba6edd..6c496edbb 100644 --- a/src/oops/interface/ObsDataVector_head.h +++ b/src/oops/interface/ObsDataVector_head.h @@ -23,6 +23,9 @@ namespace oops { template class ObsVector; // ----------------------------------------------------------------------------- +/// \brief ObsDataVector is a vector templated on data type, in the observation space +/// \details +/// oops currently uses ObsDataVector and ObsDataVector); template class ObsDataVector : public util::Printable, @@ -32,30 +35,48 @@ class ObsDataVector : public util::Printable, public: static const std::string classname() {return "oops::ObsDataVector";} - ObsDataVector(const ObsSpace &, const Variables &, const std::string name = ""); - explicit ObsDataVector(const ObsDataVector &); - explicit ObsDataVector(ObsVector &); + /// Constructor for specified ObsSpace \p os, with \p variables. If the group \p name is + /// specified, the data is read from ObsSpace for specified variables and group. + /// Otherwise ObsDataVector is allocated for specified variables and filled with zeros. + ObsDataVector(const ObsSpace & os, const Variables & vars, const std::string name = ""); + /// Copy constructor from \p other + ObsDataVector(const ObsDataVector & other); + /// Constructor from \p other ObsVector. ObsDataVector is created with variables from ObsVector + /// and assigned ObsVector values. This is only well defined for numeric DATATYPE. + explicit ObsDataVector(ObsVector & other); + /// Destructor (defined explicitly for timing and tracing) ~ObsDataVector(); -/// Interfacing + /// Accessor to the data ObsDataVec_ & obsdatavector() {return *data_;} + /// const accessor to the data const ObsDataVec_ & obsdatavector() const {return *data_;} + /// Accessor returning pointer to the data std::shared_ptr obsdatavectorptr() {return data_;} + /// const accessor returning pointer to the data std::shared_ptr obsdatavectorptr() const {return data_;} + /// Assignment operator ObsDataVector & operator = (const ObsDataVector &); + /// Zero out this ObsDataVector void zero(); - void mask(const ObsDataVector &); + /// Mask values by reading another ObsDataVector \p qc that has the same variables and contains + /// the masking information. Elements of *this* corresponding to non-zero elements of \p qc are + /// set to missing. + void mask(const ObsDataVector & qc); + /// Return the number of observations that aren't set to missing, across all MPI tasks. unsigned int nobs() const {return data_->nobs();} -// I/O - void read(const std::string &); - void save(const std::string &) const; + /// Fill ObsDataVector with data with group \p name from the associated ObsSpace + void read(const std::string & name); + /// Save this ObsDataVector as group \p name in the ObsSpace + void save(const std::string & name) const; private: void print(std::ostream &) const; + /// Pointer to the ObsDataVector implementation std::shared_ptr data_; }; diff --git a/src/oops/interface/ObsErrorBase.h b/src/oops/interface/ObsErrorBase.h new file mode 100644 index 000000000..f635b3398 --- /dev/null +++ b/src/oops/interface/ObsErrorBase.h @@ -0,0 +1,139 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_OBSERRORBASE_H_ +#define OOPS_INTERFACE_OBSERRORBASE_H_ + +#include +#include + +#include "eckit/system/ResourceUsage.h" + +#include "oops/base/ObsVector.h" +#include "oops/generic/ObsErrorBase.h" +#include "oops/interface/ObsSpace.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +namespace interface { + +// ----------------------------------------------------------------------------- +/// \brief Base class for OBS-specific implementations of the ObsError interface. +/// +/// interface::ObsErrorBase overrides oops::ObsErrorBase methods to pass OBS-specific +/// implementations of ObsVector to OBS-specific implementations of ObsError. +/// +/// Note: each subclass should typedef `Parameters_` to the name of a subclass of +/// ObsErrorParametersBase holding its configuration settings and provide a constructor with the +/// following signature: +/// +/// ObsErrorBase(const Parameters_ ¶ms, OBS::ObsSpace &obsspace, +/// const eckit::mpi::Comm &timeComm); +/// +/// This constructor should pass \c timeComm to the constructor of this class. +template +class ObsErrorBase : public oops::ObsErrorBase { + typedef typename OBS::ObsSpace ObsSpace_; + typedef typename OBS::ObsVector ObsVector_; + + public: + explicit ObsErrorBase(const eckit::mpi::Comm &timeComm) + : timeComm_(timeComm) {} + + // Overrides of oops::ObsErrorBase methods, converting between oops interface classes and + // OBS-specific classes operated upon by OBS-specific implementations of the ObsError interface. + + void multiply(oops::ObsVector &dy) const final { + this->multiply(dy.obsvector()); + } + + void inverseMultiply(oops::ObsVector &dy) const final { + this->inverseMultiply(dy.obsvector()); + } + + void randomize(oops::ObsVector &dy) const final { + this->randomize(dy.obsvector()); + } + + oops::ObsVector obserrors() const final { + return oops::ObsVector(this->getObsErrors(), timeComm_); + } + + void update(const oops::ObsVector &dy) final { + this->update(dy.obsvector()); + } + + oops::ObsVector inverseVariance() const final { + return oops::ObsVector(this->getInverseVariance(), timeComm_); + } + + // The methods below need to be overridden in subclasses (along with save(), getRMSE() and + // print(), methods inherited from parent classes that neither take nor return instances of oops + // interface classes and therefore are still abstract). + + /// Multiply a Departure \p dy by \f$R\f$. + virtual void multiply(ObsVector_ &dy) const = 0; + + /// Multiply a Departure \p dy by \f$R^{-1}\f$. + virtual void inverseMultiply(ObsVector_ &dy) const = 0; + + /// Generate a random perturbation in \p dy. + virtual void randomize(ObsVector_ &dy) const = 0; + + /// Return a copy of obs error std. dev. If this ObsVector_ is modified (e.g. by obs filters), + /// it should be passed back to update() to ensure the covariance matrix stays consistent. + virtual std::unique_ptr getObsErrors() const = 0; + + /// Set the diagonal of the covariance matrix to \p stddev squared. + virtual void update(const ObsVector_ &stddev) = 0; + + /// Return the vector of inverse obs error variances. + virtual std::unique_ptr getInverseVariance() const = 0; + + private: + const eckit::mpi::Comm & timeComm_; +}; + +// ============================================================================= + +/// \brief A subclass of ObsErrorFactory able to create instances of T (a concrete subclass of +/// interface::ObsErrorBase). Passes OBS::ObsSpace to the constructor of T. +template +class ObsErrorMaker : public ObsErrorFactory { + public: + typedef oops::ObsErrorBase ObsErrorBase_; + typedef oops::ObsErrorFactory ObsErrorFactory_; + typedef oops::ObsSpace ObsSpace_; + typedef typename T::Parameters_ Parameters_; + + explicit ObsErrorMaker(const std::string & name) : ObsErrorFactory_(name) {} + + private: + std::unique_ptr make(const ObsErrorParametersBase & parameters, + const ObsSpace_ & obsspace) override { + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return std::make_unique(stronglyTypedParameters, obsspace.obsspace(), obsspace.timeComm()); + } + + std::unique_ptr makeParameters() const override { + return std::make_unique(); + } +}; + +// ----------------------------------------------------------------------------- + +} // namespace interface + +} // namespace oops + +#endif // OOPS_INTERFACE_OBSERRORBASE_H_ diff --git a/src/oops/interface/ObsErrorCovariance.h b/src/oops/interface/ObsErrorCovariance.h deleted file mode 100644 index a47386dfb..000000000 --- a/src/oops/interface/ObsErrorCovariance.h +++ /dev/null @@ -1,142 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_INTERFACE_OBSERRORCOVARIANCE_H_ -#define OOPS_INTERFACE_OBSERRORCOVARIANCE_H_ - -#include -#include - -#include "eckit/system/ResourceUsage.h" - -#include "oops/base/ObsErrorBase.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" - -namespace eckit { - class Configuration; -} - -namespace oops { - -// ----------------------------------------------------------------------------- -/// Observation error covariance matrix -/*! - * This class provides the operations associated with the observation - * error covariance matrix. It wraps model specific observation error covariances. - */ - -template -class ObsErrorCovariance : public oops::ObsErrorBase { - typedef ObsSpace ObsSpace_; - typedef ObsVector ObsVector_; - - public: - static const std::string classname() {return "oops::ObsErrorCovariance";} - - ObsErrorCovariance(const eckit::Configuration &, const ObsSpace_ &, const Variables &); - ~ObsErrorCovariance(); - -/// Multiply a Departure by \f$R\f$ and \f$R^{-1}\f$ - void multiply(ObsVector_ &) const; - void inverseMultiply(ObsVector_ &) const; - -/// Generate random perturbation - void randomize(ObsVector_ &) const; - -/// Get mean error for Jo table - double getRMSE() const; - - private: - void print(std::ostream &) const; - std::unique_ptr covar_; -}; - -// ==================================================================================== - -template -ObsErrorCovariance::ObsErrorCovariance(const eckit::Configuration & conf, - const ObsSpace_ & obsdb, - const Variables & obsvar) : covar_() { - Log::trace() << "ObsErrorCovariance::ObsErrorCovariance starting" << std::endl; - util::Timer timer(classname(), "ObsErrorCovariance"); - size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); - covar_.reset(new OBSERR(conf, obsdb, obsvar)); - size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); - this->setObjectSize(current - init); - Log::trace() << "ObsErrorCovariance::ObsErrorCovariance done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -ObsErrorCovariance::~ObsErrorCovariance() { - Log::trace() << "ObsErrorCovariance::~ObsErrorCovariance starting" << std::endl; - util::Timer timer(classname(), "~ObsErrorCovariance"); - covar_.reset(); - Log::trace() << "ObsErrorCovariance::~ObsErrorCovariance done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsErrorCovariance::multiply(ObsVector_ & dy) const { - Log::trace() << "ObsErrorCovariance::multiply starting" << std::endl; - util::Timer timer(classname(), "multiply"); - covar_->multiply(dy.obsvector()); - Log::trace() << "ObsErrorCovariance::multiply done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsErrorCovariance::inverseMultiply(ObsVector_ & dy) const { - Log::trace() << "ObsErrorCovariance::inverseMultiply starting" << std::endl; - util::Timer timer(classname(), "inverseMultiply"); - covar_->inverseMultiply(dy.obsvector()); - Log::trace() << "ObsErrorCovariance::inverseMultiply done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsErrorCovariance::randomize(ObsVector_ & dy) const { - Log::trace() << "ObsErrorCovariance::randomize starting" << std::endl; - util::Timer timer(classname(), "randomize"); - covar_->randomize(dy.obsvector()); - Log::trace() << "ObsErrorCovariance::randomize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -double ObsErrorCovariance::getRMSE() const { - Log::trace() << "ObsErrorCovariance::getRMSE starting" << std::endl; - util::Timer timer(classname(), "getRMSE"); - double zz = covar_->getRMSE(); - Log::trace() << "ObsErrorCovariance::getRMSE done" << std::endl; - return zz; -} - -// ----------------------------------------------------------------------------- - -template -void ObsErrorCovariance::print(std::ostream & os) const { - Log::trace() << "ObsErrorCovariance::print starting" << std::endl; - util::Timer timer(classname(), "print"); - os << *covar_; - Log::trace() << "ObsErrorCovariance::print done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_INTERFACE_OBSERRORCOVARIANCE_H_ diff --git a/src/oops/interface/ObsFilter.h b/src/oops/interface/ObsFilter.h deleted file mode 100644 index 114c310c5..000000000 --- a/src/oops/interface/ObsFilter.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - * (C) Copyright 2017-2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_INTERFACE_OBSFILTER_H_ -#define OOPS_INTERFACE_OBSFILTER_H_ - -#include -#include - -#include "eckit/config/LocalConfiguration.h" -#include "oops/base/ObsFilterBase.h" -#include "oops/base/Variables.h" -#include "oops/interface/GeoVaLs.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsDiagnostics.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/util/dot_product.h" -#include "oops/util/Logger.h" -#include "oops/util/parameters/HasParameters_.h" -#include "oops/util/parameters/ParametersOrConfiguration.h" - -namespace oops { - -// ----------------------------------------------------------------------------- - -/// Note: implementations of this interface can opt to extract their settings either from -/// a Configuration object or from a subclass of ObsFilterParametersBase. -/// -/// In the former case, they should provide a constructor with the following signature: -/// -/// ObsFilter(const ObsSpace_ &, const eckit::Configuration &, -/// ObsDataPtr_, ObsDataPtr_); -/// -/// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase -/// holding the settings of the filter in question. The implementation of the ObsFilter interface -/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with -/// the following signature: -/// -/// ObsFilter(const ObsSpace_ &, const Parameters_ &, -/// ObsDataPtr_, ObsDataPtr_); -template -class ObsFilter : public ObsFilterBase { - typedef GeoVaLs GeoVaLs_; - typedef ObsDiagnostics ObsDiags_; - typedef ObsSpace ObsSpace_; - typedef ObsVector ObsVector_; - template using ObsDataPtr_ = std::shared_ptr >; - template using ObsDataVec_ = typename OBS::template ObsDataVector; - - public: - /// Defined as FILTER::Parameters_ if FILTER defines a Parameters_ type; otherwise as - /// GenericObsFilterParameters - typedef TParameters_IfAvailableElseFallbackType_t Parameters_; - - static const std::string classname() {return "oops::ObsFilter";} - - ObsFilter(const ObsSpace_ &, const Parameters_ &, - ObsDataPtr_, ObsDataPtr_); - ObsFilter(const ObsSpace_ &, const eckit::Configuration &, - ObsDataPtr_, ObsDataPtr_); - ~ObsFilter(); - - void preProcess() const override; - void priorFilter(const GeoVaLs_ &) const override; - void postFilter(const ObsVector_ &, const ObsDiags_ &) const override; - - Variables requiredVars() const override; - Variables requiredHdiagnostics() const override; - - private: - void print(std::ostream &) const override; - - const ObsSpace_ & obsdb_; - const std::unique_ptr parameters_; - std::unique_ptr ofilt_; -}; - -// ----------------------------------------------------------------------------- - -template -ObsFilter::ObsFilter(const ObsSpace_ & os, - const Parameters_ & parameters, - ObsDataPtr_ flags, ObsDataPtr_ obserr) - : obsdb_(os), parameters_(parameters.clone()), ofilt_() -{ - Log::trace() << "ObsFilter::ObsFilter Configuration starting" << std::endl; - util::Timer timer(classname(), "ObsFilter"); - - std::shared_ptr > qc; - std::shared_ptr > oberr; - if (flags) qc = flags->obsdatavectorptr(); - if (obserr) oberr = obserr->obsdatavectorptr(); - - ofilt_.reset(new FILTER(obsdb_.obsspace(), - parametersOrConfiguration::value>(parameters), - qc, oberr)); - Log::trace() << "ObsFilter::ObsFilter Configuration done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -ObsFilter::ObsFilter(const ObsSpace_ & os, - const eckit::Configuration & conf, - ObsDataPtr_ flags, ObsDataPtr_ obserr) - : ObsFilter(os, validateAndDeserialize(conf), flags, obserr) -{} - -// ----------------------------------------------------------------------------- - -template -ObsFilter::~ObsFilter() { - Log::trace() << "ObsFilter::~ObsFilter starting" << std::endl; - util::Timer timer(classname(), "~ObsFilter"); - ofilt_.reset(); - Log::trace() << "ObsFilter::~ObsFilter done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsFilter::preProcess() const { - Log::trace() << "ObsFilter:: preProcess starting" << std::endl; - util::Timer timer(classname(), "preProcess"); - ofilt_->preProcess(); - Log::trace() << "ObsFilter:: preProcess done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsFilter::priorFilter(const GeoVaLs_ & gv) const { - Log::trace() << "ObsFilter:: priorFilter starting" << std::endl; - util::Timer timer(classname(), "priorFilter"); - ofilt_->priorFilter(gv.geovals()); - Log::trace() << "ObsFilter:: priorFilter done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void ObsFilter::postFilter(const ObsVector_ & ov, const ObsDiags_ & dv) const { - Log::trace() << "ObsFilter::postFilter starting" << std::endl; - util::Timer timer(classname(), "postFilter"); - ofilt_->postFilter(ov.obsvector(), dv.obsdiagnostics()); - Log::trace() << "ObsFilter::postFilter done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -Variables ObsFilter::requiredVars() const { - Log::trace() << "ObsFilter::requiredVars" << std::endl; - return ofilt_->requiredVars(); -} - -// ----------------------------------------------------------------------------- - -template -Variables ObsFilter::requiredHdiagnostics() const { - Log::trace() << "ObsFilter::requiredHdiagnostics" << std::endl; - return ofilt_->requiredHdiagnostics(); -} - -// ----------------------------------------------------------------------------- - -template -void ObsFilter::print(std::ostream & os) const { - os << "ObsFilter " << *parameters_; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_INTERFACE_OBSFILTER_H_ diff --git a/src/oops/interface/ObsFilterBase.h b/src/oops/interface/ObsFilterBase.h new file mode 100644 index 000000000..8c5ff5867 --- /dev/null +++ b/src/oops/interface/ObsFilterBase.h @@ -0,0 +1,126 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_OBSFILTERBASE_H_ +#define OOPS_INTERFACE_OBSFILTERBASE_H_ + +#include +#include + +#include "oops/base/ObsVector.h" +#include "oops/generic/ObsFilterBase.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsDiagnostics.h" +#include "oops/interface/ObsSpace.h" + +namespace oops { + +namespace interface { + +// ----------------------------------------------------------------------------- + +/// \brief Base class for OBS-specific implementations of the ObsFilter interface. +/// interface::ObsFilterBase overrides oops::ObsFilterBase methods to pass OBS-specific +/// implementations of GeoVaLs, ObsDiags, ObsSpace and ObsVector to the OBS-specific +/// implementation of ObsFilterBase. +/// +/// Note: implementations of the ObsFilter interface can opt to extract their settings either from +/// a Configuration object or from a subclass of ObsFilterParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const eckit::Configuration &, +/// ObsDataPtr_, ObsDataPtr_); +/// +/// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase +/// holding the settings of the filter in question. The implementation of the ObsFilter interface +/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with +/// the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const Parameters_ &, +/// ObsDataPtr_, ObsDataPtr_); +template +class ObsFilterBase : public oops::ObsFilterBase { + typedef typename OBS::GeoVaLs GeoVaLs_; + typedef typename OBS::ObsDiagnostics ObsDiags_; + typedef typename OBS::ObsSpace ObsSpace_; + typedef typename OBS::ObsVector ObsVector_; + + public: + // Overrides of oops::ObsFilterBase methods, converting between oops interface classes and + // OBS-specific classes operated upon by OBS-specific implementations of the ObsFilter interface. + + void priorFilter(const GeoVaLs &gv) final { + this->priorFilter(gv.geovals()); + } + + void postFilter(const oops::ObsVector &ov, const oops::ObsVector &bv, + const ObsDiagnostics &dv) final { + this->postFilter(ov.obsvector(), bv.obsvector(), dv.obsdiagnostics()); + } + + // The methods below need to be overridden in subclasses (along with preProcess(), requiredVars() + // and requiredHdiagnostics() and print(), methods inherited from parent classes that neither + // take nor return instances of oops interface classes and therefore remain abstract). + + /// \brief Perform any observation processing steps that require access to GeoVaLs, but not to + /// outputs produced by the observation operator. + virtual void priorFilter(const GeoVaLs_ &gv) = 0; + + /// \brief Perform any observation processing steps that require access to + /// outputs produced by the observation operator. + /// + /// \param ov + /// Model equivalents produced by the observation operator. + /// \param bias + /// Bias of departure produced by the observation operator. + /// \param dv + /// Observation diagnostics produced by the observation operator. + virtual void postFilter(const ObsVector_ &ov, const ObsVector_ &bv, const ObsDiags_ &dv) = 0; +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of FilterFactory able to create instances of T (a concrete subclass of +/// interface::ObsFilterBase). Passes OBS::ObsSpace to the constructor of T. +template +class FilterMaker : public FilterFactory { + private: + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericObsFilterParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + public: + typedef oops::ObsSpace ObsSpace_; + template using ObsDataPtr_ = std::shared_ptr >; + + explicit FilterMaker(const std::string & name) : FilterFactory(name) {} + + std::unique_ptr> make(const ObsSpace_ & os, + const ObsFilterParametersBase & params, + ObsDataPtr_ & flags, + ObsDataPtr_ & obserr) override { + const auto &stronglyTypedParams = dynamic_cast(params); + const auto ¶msOrConfig = + parametersOrConfiguration::value>(stronglyTypedParams); + return std::make_unique(os.obsspace(), + paramsOrConfig, + flags ? flags->obsdatavectorptr() : nullptr, + obserr ? obserr->obsdatavectorptr() : nullptr); + } + + std::unique_ptr makeParameters() const override { + return std::make_unique(); + } +}; + +} // namespace interface + +} // namespace oops + +#endif // OOPS_INTERFACE_OBSFILTERBASE_H_ diff --git a/src/oops/interface/ObsLocalization.h b/src/oops/interface/ObsLocalization.h index 39f169568..56e765104 100644 --- a/src/oops/interface/ObsLocalization.h +++ b/src/oops/interface/ObsLocalization.h @@ -14,9 +14,8 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsLocalizationBase.h" -#include "oops/interface/ObsDataVector.h" +#include "oops/base/ObsVector.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" namespace oops { @@ -31,7 +30,6 @@ class ObsLocalization : public util::Printable, typedef ObsLocalizationBase ObsLocBase_; typedef GeometryIterator GeometryIterator_; typedef ObsSpace ObsSpace_; - typedef ObsDataVector ObsDataVector_; typedef ObsVector ObsVector_; public: @@ -40,12 +38,11 @@ class ObsLocalization : public util::Printable, ObsLocalization(const eckit::Configuration &, const ObsSpace_ &); ~ObsLocalization(); - /// compute obs-space localization: fill \p obsvector with observation-space - /// localization values between observations and \p point in model-space, and - /// fill \p outside with flags on whether obs is local or not (1: outside of - /// localization, 0: inside of localization, local) + /// compute obs-space localization: fill \p locfactor with observation-space + /// localization values between observations and \p point in model-space. + /// Set \p locfactor to missing value for observations that are not local. void computeLocalization(const GeometryIterator_ & point, - ObsDataVector_ & outside, ObsVector_ & obsvector) const override; + ObsVector_ & locfactor) const override; private: void print(std::ostream &) const override; @@ -80,10 +77,10 @@ ObsLocalization::~ObsLocalization() { template void ObsLocalization::computeLocalization(const GeometryIterator_ & p, - ObsDataVector_ & local, ObsVector_ & obsvector) const { + ObsVector_ & locfactor) const { Log::trace() << "ObsLocalization:: computeLocalization starting" << std::endl; util::Timer timer(classname(), "computeLocalization"); - obsloc_->computeLocalization(p, local, obsvector); + obsloc_->computeLocalization(p, locfactor); Log::trace() << "ObsLocalization:: computeLocalization done" << std::endl; } diff --git a/src/oops/interface/ObsOperator.h b/src/oops/interface/ObsOperator.h index 86611aa95..d32eef7ea 100644 --- a/src/oops/interface/ObsOperator.h +++ b/src/oops/interface/ObsOperator.h @@ -16,12 +16,12 @@ #include +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsAuxControl.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -30,7 +30,17 @@ namespace oops { // ----------------------------------------------------------------------------- - +/// \brief MODEL-agnostic part of nonlinear observation (forward) operator. +/// The full nonlinear observation operator from State x to ObsVector is: +/// ObsOperator ( GetValues (State) ) +/// ObsOperator uses GeoVaLs (result of GetValues(State) - model State at +/// observations locations) as input data to compute forward operator. +/// +/// Note: each implementation should typedef `Parameters_` to the name of a subclass of +/// oops::Parameters holding its configuration settings and provide a constructor with the +/// following signature: +/// +/// ObsOperator(const OBS::ObsSpace &, const Parameters_ &); template class ObsOperator : public util::Printable, private boost::noncopyable, @@ -44,23 +54,41 @@ class ObsOperator : public util::Printable, typedef ObsSpace ObsSpace_; public: + /// A subclass of oops::Parameters holding the configuration settings of the operator. + typedef typename ObsOperator_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsOperator";} - ObsOperator(const ObsSpace_ &, const eckit::Configuration &); + /// Set up observation operator for the \p obsspace observations, with + /// parameters defined in \p parameters + ObsOperator(const ObsSpace_ & obsspace, const Parameters_ & parameters); ~ObsOperator(); -/// Obs Operator - void simulateObs(const GeoVaLs_ &, ObsVector_ &, const ObsAuxControl_ &, ObsDiags_ &) const; - -/// Interfacing - const ObsOperator_ & obsoperator() const {return *oper_;} - -/// Other - const Variables & requiredVars() const; // Required input variables from Model + /// Compute forward operator \p y = ObsOperator (\p x). + /// \param[in] x obs operator input, State interpolated to observations locations. + /// \param[out] y result of computing obs operator on \p x. + /// \param[in] obsaux additional input for computing H(x), used in the minimization + /// in Variational DA, e.g. bias correction coefficients or obs operator + /// parameters. + /// \param[out] obsbias bias correction of the departure between \p y and the observed values; + /// when \p obsbias is non-zero, it is added to \p y within the obs + /// operator + /// \param[out] obsdiags additional diagnostics output from computing obs operator that is not + /// used in the assimilation, and can be used by ObsFilters. + void simulateObs(const GeoVaLs_ & x_int, ObsVector_ & y, const ObsAuxControl_ & obsaux, + ObsVector_ & obsbias, ObsDiags_ & obsdiags) const; + + /// Variables required from the model State to compute obs operator. These variables + /// will be provided in GeoVaLs passed to simulateObs. + const Variables & requiredVars() const; + /// Locations used for computing GeoVaLs that will be passed to simulateObs. Locations_ locations() const; private: + /// Print, used for logging void print(std::ostream &) const; + + /// Pointer to the implementation of ObsOperator std::unique_ptr oper_; }; @@ -68,10 +96,10 @@ class ObsOperator : public util::Printable, template ObsOperator::ObsOperator(const ObsSpace_ & os, - const eckit::Configuration & config) : oper_() { + const Parameters_ & parameters) : oper_() { Log::trace() << "ObsOperator::ObsOperator starting" << std::endl; util::Timer timer(classname(), "ObsOperator"); - oper_.reset(new ObsOperator_(os.obsspace(), config)); + oper_.reset(new ObsOperator_(os.obsspace(), parameters)); Log::trace() << "ObsOperator::ObsOperator done" << std::endl; } @@ -89,10 +117,12 @@ ObsOperator::~ObsOperator() { template void ObsOperator::simulateObs(const GeoVaLs_ & gvals, ObsVector_ & yy, - const ObsAuxControl_ & aux, ObsDiags_ & ydiag) const { + const ObsAuxControl_ & aux, ObsVector_ & ybias, + ObsDiags_ & ydiag) const { Log::trace() << "ObsOperator::simulateObs starting" << std::endl; util::Timer timer(classname(), "simulateObs"); - oper_->simulateObs(gvals.geovals(), yy.obsvector(), aux.obsauxcontrol(), ydiag.obsdiagnostics()); + oper_->simulateObs(gvals.geovals(), yy.obsvector(), aux.obsauxcontrol(), ybias.obsvector(), + ydiag.obsdiagnostics()); Log::trace() << "ObsOperator::simulateObs done" << std::endl; } diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 5d4cb7edd..eeae08b57 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -42,13 +42,15 @@ namespace oops { template class ObsSpace : public util::Printable, private util::ObjectCounter > { - typedef typename OBS::ObsSpace ObsSpace_; - typedef GeometryIterator ObsIterator_; + typedef typename OBS::ObsSpace ObsSpace_; + typedef GeometryIterator ObsIterator_; public: + typedef typename ObsSpace_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsSpace";} - ObsSpace(const eckit::Configuration &, const eckit::mpi::Comm &, + ObsSpace(const Parameters_ &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm & time = oops::mpi::myself()); ~ObsSpace(); @@ -87,7 +89,7 @@ class ObsSpace : public util::Printable, // ----------------------------------------------------------------------------- template -ObsSpace::ObsSpace(const eckit::Configuration & conf, +ObsSpace::ObsSpace(const Parameters_ & params, const eckit::mpi::Comm & comm, const util::DateTime & bgn, const util::DateTime & end, @@ -95,7 +97,7 @@ ObsSpace::ObsSpace(const eckit::Configuration & conf, Log::trace() << "ObsSpace::ObsSpace starting" << std::endl; util::Timer timer(classname(), "ObsSpace"); size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); - obsdb_.reset(new ObsSpace_(conf, comm, bgn, end, time)); + obsdb_.reset(new ObsSpace_(params, comm, bgn, end, time)); size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); this->setObjectSize(current - init); Log::trace() << "ObsSpace::ObsSpace done" << std::endl; diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index 1c24a081f..a701db7e2 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "oops/interface/ObsDataVector_head.h" #include "oops/interface/ObsSpace.h" @@ -35,6 +36,10 @@ namespace util { namespace oops { +namespace interface { + +// ----------------------------------------------------------------------------- +/// \brief Holds observation vector (e.g. vector of observation values, or of computed H(x)) // ----------------------------------------------------------------------------- template @@ -48,57 +53,75 @@ class ObsVector : public util::Printable, /// Creates vector from \p obsspace. If \p name is specified, reads the /// specified \p name variable from \p obsspace. Otherwise, zero vector is created. explicit ObsVector(const ObsSpace & obsspace, const std::string name = ""); + + /// Wraps an existing ObsVector_. + /// This wrapping constructor doesn't need to be implemented in an ObsVector implementation. + /// \param obsvector The vector to wrap. + explicit ObsVector(std::unique_ptr obsvector); + /// Copy constructor ObsVector(const ObsVector &); + /// Destructor (defined explicitly for timing and tracing) ~ObsVector(); -/// Interfacing + /// Accessor ObsVector_ & obsvector() {return *data_;} + /// Const accessor const ObsVector_ & obsvector() const {return *data_;} -// Linear algebra + /// Linear algebra operators ObsVector & operator = (const ObsVector &); ObsVector & operator*= (const double &); ObsVector & operator+= (const ObsVector &); ObsVector & operator-= (const ObsVector &); ObsVector & operator*= (const ObsVector &); ObsVector & operator/= (const ObsVector &); + /// Add \p zz * \p rhs to the ObsVector. + void axpy(const double & zz, const ObsVector & rhs); /// Pack observations local to this MPI task into an Eigen vector - /// (excluding vector elements that are masked out and where \p mask != 0) - Eigen::VectorXd packEigen(const ObsDataVector & mask) const; + /// (excluding vector elements that are masked out: where \p mask is a missing value) + Eigen::VectorXd packEigen(const ObsVector & mask) const; /// Number of non-masked out observations local to this MPI task /// (size of an Eigen vector returned by `packEigen`) - size_t packEigenSize(const ObsDataVector & mask) const; + size_t packEigenSize(const ObsVector & mask) const; + /// Zero out this ObsVector void zero(); /// Set this ObsVector to ones (used in tests) void ones(); - void axpy(const double &, const ObsVector &); + /// Set each value in this ObsVector to its inverse void invert(); + /// Set each value in this ObsVector to a random value void random(); - double dot_product_with(const ObsVector &) const; - double rms() const; - /// Mask out elements of the vector where the passed in flags are > 0 - void mask(const ObsDataVector &); - ObsVector & operator =(const ObsDataVector &); -// I/O + /// Return the dot product between this ObsVector and another one \p other + double dot_product_with(const ObsVector & other) const; + /// Return this ObsVector rms + double rms() const; + /// Mask out elements of the vector where \p mask is > 0 + void mask(const ObsDataVector & mask); + /// Mask out elements of the vector where \p mask is a missing value + void mask(const ObsVector & mask); + /// Assignment operator from \p rhs ObsDataVector + ObsVector & operator =(const ObsDataVector & rhs); + + /// Save this ObsVector as group \p name in the ObsSpace void save(const std::string &) const; + /// Fill ObsVector with data with group \p name from the associated ObsSpace void read(const std::string &); - /// number of non-masked out observations (across all MPI tasks) + /// Number of non-masked out observations (across all MPI tasks) unsigned int nobs() const; private: void print(std::ostream &) const; std::unique_ptr data_; - const eckit::mpi::Comm & commTime_; }; // ----------------------------------------------------------------------------- template ObsVector::ObsVector(const ObsSpace & os, const std::string name) - : data_(), commTime_(os.timeComm()) { + : data_() { Log::trace() << "ObsVector::ObsVector starting " << name << std::endl; util::Timer timer(classname(), "ObsVector"); data_.reset(new ObsVector_(os.obsspace(), name)); @@ -107,7 +130,16 @@ ObsVector::ObsVector(const ObsSpace & os, const std::string name) } // ----------------------------------------------------------------------------- template -ObsVector::ObsVector(const ObsVector & other): data_(), commTime_(other.commTime_) { +ObsVector::ObsVector(std::unique_ptr obsvector) + : data_(std::move(obsvector)) { + Log::trace() << "ObsVector::ObsVector starting " << std::endl; + util::Timer timer(classname(), "ObsVector"); + this->setObjectSize(data_->size() * sizeof(double)); + Log::trace() << "ObsVector::ObsVector done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +ObsVector::ObsVector(const ObsVector & other): data_() { Log::trace() << "ObsVector::ObsVector starting" << std::endl; util::Timer timer(classname(), "ObsVector"); data_.reset(new ObsVector_(*other.data_)); @@ -245,7 +277,6 @@ double ObsVector::dot_product_with(const ObsVector & other) const { util::Timer timer(classname(), "dot_product"); double zz = data_->dot_product_with(*other.data_); - commTime_.allReduceInPlace(zz, eckit::mpi::Operation::SUM); Log::trace() << "ObsVector::dot_product done" << std::endl; return zz; @@ -260,6 +291,14 @@ void ObsVector::mask(const ObsDataVector & qc) { } // ----------------------------------------------------------------------------- template +void ObsVector::mask(const ObsVector & mask) { + Log::trace() << "ObsVector::mask(ObsVector) starting" << std::endl; + util::Timer timer(classname(), "mask(ObsVector)"); + data_->mask(mask.obsvector()); + Log::trace() << "ObsVector::mask(ObsVector) done" << std::endl; +} +// ----------------------------------------------------------------------------- +template ObsVector & ObsVector::operator=(const ObsDataVector & rhs) { Log::trace() << "ObsVector::operator= starting" << std::endl; util::Timer timer(classname(), "operator="); @@ -273,15 +312,7 @@ double ObsVector::rms() const { Log::trace() << "ObsVector::rms starting" << std::endl; util::Timer timer(classname(), "rms"); - double zz = 0.0; - size_t ntot = this->nobs(); - if (ntot > 0) { - zz = data_->rms(); - double zzz = zz * zz * static_cast(data_->nobs()); - commTime_.allReduceInPlace(zzz, eckit::mpi::Operation::SUM); - zzz /= static_cast(ntot); - zz = std::sqrt(zzz); - } + double zz = data_->rms(); Log::trace() << "ObsVector::rms done" << std::endl; return zz; @@ -290,7 +321,6 @@ double ObsVector::rms() const { template unsigned int ObsVector::nobs() const { int nobs = data_->nobs(); - commTime_.allReduceInPlace(nobs, eckit::mpi::Operation::SUM); return nobs; } // ----------------------------------------------------------------------------- @@ -298,11 +328,7 @@ template void ObsVector::print(std::ostream & os) const { Log::trace() << "ObsVector::print starting" << std::endl; util::Timer timer(classname(), "print"); - if (commTime_.size() > 1) { - gatherPrint(os, *data_, commTime_); - } else { - os << *data_; - } + os << *data_; Log::trace() << "ObsVector::print done" << std::endl; } // ----------------------------------------------------------------------------- @@ -317,22 +343,22 @@ void ObsVector::save(const std::string & name) const { } // ----------------------------------------------------------------------------- template -Eigen::VectorXd ObsVector::packEigen(const ObsDataVector & mask) const { +Eigen::VectorXd ObsVector::packEigen(const ObsVector & mask) const { Log::trace() << "ObsVector::packEigen starting " << std::endl; util::Timer timer(classname(), "packEigen"); - Eigen::VectorXd vec = data_->packEigen(mask.obsdatavector()); + Eigen::VectorXd vec = data_->packEigen(mask.obsvector()); Log::trace() << "ObsVector::packEigen done" << std::endl; return vec; } // ----------------------------------------------------------------------------- template -size_t ObsVector::packEigenSize(const ObsDataVector & mask) const { +size_t ObsVector::packEigenSize(const ObsVector & mask) const { Log::trace() << "ObsVector::packEigenSize starting " << std::endl; util::Timer timer(classname(), "packEigenSize"); - size_t len = data_->packEigenSize(mask.obsdatavector()); + size_t len = data_->packEigenSize(mask.obsvector()); Log::trace() << "ObsVector::packEigen done" << std::endl; return len; @@ -349,6 +375,8 @@ void ObsVector::read(const std::string & name) { } // ----------------------------------------------------------------------------- +} // namespace interface + } // namespace oops #endif // OOPS_INTERFACE_OBSVECTOR_H_ diff --git a/src/oops/interface/State.h b/src/oops/interface/State.h index 37deb13c7..ac3d62442 100644 --- a/src/oops/interface/State.h +++ b/src/oops/interface/State.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. - * (C) Copyright 2017-2019 UCAR. + * (C) Copyright 2017-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -17,9 +17,9 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" #include "oops/base/PostProcessor.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/gatherPrint.h" #include "oops/util/ObjectCounter.h" @@ -29,6 +29,8 @@ namespace oops { +namespace interface { + /// Encapsulates the model state // ----------------------------------------------------------------------------- @@ -37,55 +39,67 @@ template class State : public util::Printable, public util::Serializable, private util::ObjectCounter > { - typedef typename MODEL::State State_; - typedef Geometry Geometry_; + typedef typename MODEL::State State_; + typedef oops::Geometry Geometry_; public: static const std::string classname() {return "oops::State";} -/// Constructor, destructor - State(const Geometry_ &, const Variables &, const util::DateTime &); - State(const Geometry_ &, const eckit::Configuration &); - State(const Geometry_ &, const State &); + /// Constructor for specified \p resol, with \p vars, valid at \p time + State(const Geometry_ & resol, const Variables & vars, const util::DateTime & time); + /// Constructor for specified \p resol and files read from \p conf + State(const Geometry_ & resol, const eckit::Configuration & conf); + /// Copies \p other State, changing its resolution to \p geometry + State(const Geometry_ & resol, const State & other); + /// Copy constructor State(const State &); + /// Destructor (defined explicitly for timing and tracing) ~State(); - State & operator=(const State &); // Is that used anywhere? + /// Assignment operator + State & operator =(const State &); -/// Interfacing + /// Accessor State_ & state() {return *state_;} + /// const accessor const State_ & state() const {return *state_;} -/// Time + /// Accessor to the time of this State const util::DateTime validTime() const {return state_->validTime();} + /// Update this State's valid time by \p dt void updateTime(const util::Duration & dt) {state_->updateTime(dt);} -/// I/O and diagnostics + /// Read this State from file void read(const eckit::Configuration &); + /// Write this State out to file void write(const eckit::Configuration &) const; - double norm() const; // Only for tests + /// Norm (used in tests) + double norm() const; + + /// Accessor to geometry associated with this State Geometry_ geometry() const; + /// Accessor to variables associated with this State const Variables & variables() const; -/// Accumulator + /// Zero out this State void zero(); - void accumul(const double &, const State &); + /// Accumulate (add \p w * \p x to the state) + void accumul(const double & w, const State & x); -/// Serialize and deserialize + /// Serialize and deserialize (used in 4DEnVar, weak-constraint 4DVar and Block-Lanczos minimizer) size_t serialSize() const override; void serialize(std::vector &) const override; void deserialize(const std::vector &, size_t &) override; private: - void print(std::ostream &) const override; std::unique_ptr state_; - const eckit::mpi::Comm & commTime_; + void print(std::ostream &) const override; }; // ============================================================================= template State::State(const Geometry_ & resol, const Variables & vars, - const util::DateTime & time) : state_(), commTime_(resol.timeComm()) + const util::DateTime & time) : state_() { Log::trace() << "State::State starting" << std::endl; util::Timer timer(classname(), "State"); @@ -98,7 +112,7 @@ State::State(const Geometry_ & resol, const Variables & vars, template State::State(const Geometry_ & resol, const eckit::Configuration & conf) - : state_(), commTime_(resol.timeComm()) + : state_() { Log::trace() << "State::State read starting" << std::endl; util::Timer timer(classname(), "State"); @@ -124,7 +138,7 @@ State::State(const Geometry_ & resol, const eckit::Configuration & conf) template State::State(const Geometry_ & resol, const State & other) - : state_(), commTime_(resol.timeComm()) + : state_() { Log::trace() << "State::State interpolated starting" << std::endl; util::Timer timer(classname(), "State"); @@ -136,7 +150,7 @@ State::State(const Geometry_ & resol, const State & other) // ----------------------------------------------------------------------------- template -State::State(const State & other) : state_(), commTime_(other.commTime_) +State::State(const State & other) : state_() { Log::trace() << "State::State starting copy" << std::endl; util::Timer timer(classname(), "State"); @@ -193,9 +207,6 @@ double State::norm() const { Log::trace() << "State::norm starting" << std::endl; util::Timer timer(classname(), "norm"); double zz = state_->norm(); - zz *= zz; - commTime_.allReduceInPlace(zz, eckit::mpi::Operation::SUM); - zz = sqrt(zz); Log::trace() << "State::norm done" << std::endl; return zz; } @@ -203,10 +214,10 @@ double State::norm() const { // ----------------------------------------------------------------------------- template -Geometry State::geometry() const { +oops::Geometry State::geometry() const { Log::trace() << "State::geometry starting" << std::endl; util::Timer timer(classname(), "geometry"); - Geometry geom(state_->geometry()); + oops::Geometry geom(state_->geometry()); Log::trace() << "State::geometry done" << std::endl; return geom; } @@ -255,11 +266,7 @@ template void State::print(std::ostream & os) const { Log::trace() << "State::print starting" << std::endl; util::Timer timer(classname(), "print"); - if (commTime_.size() > 1) { - gatherPrint(os, *state_, commTime_); - } else { - os << *state_; - } + os << *state_; Log::trace() << "State::print done" << std::endl; } @@ -285,6 +292,8 @@ void State::accumul(const double & zz, const State & xx) { // ----------------------------------------------------------------------------- +} // namespace interface + } // namespace oops #endif // OOPS_INTERFACE_STATE_H_ diff --git a/src/oops/interface/VariableChange.h b/src/oops/interface/VariableChange.h index 16df20c00..787ea18fe 100644 --- a/src/oops/interface/VariableChange.h +++ b/src/oops/interface/VariableChange.h @@ -11,11 +11,11 @@ #include #include +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/VariableChangeBase.h" #include "oops/base/VariableChangeParametersBase.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" diff --git a/src/oops/runs/AddIncrement.h b/src/oops/runs/AddIncrement.h index 3ac377841..f777641a8 100644 --- a/src/oops/runs/AddIncrement.h +++ b/src/oops/runs/AddIncrement.h @@ -12,9 +12,9 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/ConvertIncrement.h b/src/oops/runs/ConvertIncrement.h index 60802a0c6..e4a969137 100644 --- a/src/oops/runs/ConvertIncrement.h +++ b/src/oops/runs/ConvertIncrement.h @@ -13,11 +13,11 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/State.h" #include "oops/generic/instantiateVariableChangeFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/ConvertState.h b/src/oops/runs/ConvertState.h index 6428c3fc4..53386915c 100644 --- a/src/oops/runs/ConvertState.h +++ b/src/oops/runs/ConvertState.h @@ -13,9 +13,9 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/generic/instantiateVariableChangeFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" diff --git a/src/oops/runs/DiffStates.h b/src/oops/runs/DiffStates.h index 24c0eca33..7a397ad31 100644 --- a/src/oops/runs/DiffStates.h +++ b/src/oops/runs/DiffStates.h @@ -13,9 +13,9 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/Dirac.h b/src/oops/runs/Dirac.h index fa48ec28b..3798cf6ba 100644 --- a/src/oops/runs/Dirac.h +++ b/src/oops/runs/Dirac.h @@ -17,14 +17,14 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/IncrementEnsemble.h" #include "oops/base/instantiateCovarFactory.h" #include "oops/base/ModelSpaceCovarianceBase.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -37,7 +37,7 @@ template class Dirac : public Application { typedef Geometry Geometry_; typedef Increment Increment_; typedef State State_; - typedef LocalizationBase Localization_; + typedef Localization Localization_; typedef IncrementEnsemble Ensemble_; typedef std::shared_ptr> EnsemblePtr_; @@ -139,11 +139,10 @@ template class Dirac : public Application { dxdir.dirac(diracConfig); // Setup localization - std::unique_ptr loc_ = - LocalizationFactory::create(resol, time, locConfigs[jcomp]); + Localization_ loc_(resol, locConfigs[jcomp]); // Apply localization - loc_->multiply(dxdir); + loc_.multiply(dxdir); // Write increment const eckit::LocalConfiguration output_localization(fullConfig, "output localization"); diff --git a/src/oops/runs/EnsRecenter.h b/src/oops/runs/EnsRecenter.h index c1a834fdb..b2f338f2f 100644 --- a/src/oops/runs/EnsRecenter.h +++ b/src/oops/runs/EnsRecenter.h @@ -14,10 +14,10 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/EnsVariance.h b/src/oops/runs/EnsVariance.h index 986e7cd07..e235b920c 100644 --- a/src/oops/runs/EnsVariance.h +++ b/src/oops/runs/EnsVariance.h @@ -14,10 +14,10 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" #include "oops/base/IncrementEnsemble.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/ExternalDFI.h b/src/oops/runs/ExternalDFI.h index 276a7ea05..c4682b3a2 100644 --- a/src/oops/runs/ExternalDFI.h +++ b/src/oops/runs/ExternalDFI.h @@ -16,15 +16,15 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" +#include "oops/base/Geometry.h" +#include "oops/base/Model.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/StateWriter.h" #include "oops/base/Variables.h" #include "oops/base/WeightedMean.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/Forecast.h b/src/oops/runs/Forecast.h index 43a7334a0..4f35c5a3a 100644 --- a/src/oops/runs/Forecast.h +++ b/src/oops/runs/Forecast.h @@ -14,13 +14,13 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Model.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/StateWriter.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/GenEnsPertB.h b/src/oops/runs/GenEnsPertB.h index 8b9530951..5b58b6631 100644 --- a/src/oops/runs/GenEnsPertB.h +++ b/src/oops/runs/GenEnsPertB.h @@ -17,16 +17,16 @@ #include "eckit/config/Configuration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/instantiateCovarFactory.h" +#include "oops/base/Model.h" #include "oops/base/ModelSpaceCovarianceBase.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateWriter.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" diff --git a/src/oops/runs/HofX3D.h b/src/oops/runs/HofX3D.h index 905ff4112..9c4dffaf6 100644 --- a/src/oops/runs/HofX3D.h +++ b/src/oops/runs/HofX3D.h @@ -15,17 +15,17 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CalcHofX.h" +#include "oops/base/Geometry.h" #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/ChangeVariables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/ConfigFunctions.h" @@ -110,7 +110,8 @@ template class HofX3D : public Application { // loop over all observation types for (size_t jj = 0; jj < obspaces.size(); ++jj) { GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); - geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); + geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj], + geometry.variableSizes(vars[jj]))); getvals.fillGeoVaLs(zz, winbgn, winend, *geovals[jj]); } diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index b5d789b30..99692613f 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -13,30 +13,109 @@ #define OOPS_RUNS_HOFX4D_H_ #include +#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" +#include "oops/base/Geometry.h" #include "oops/base/instantiateObsFilterFactory.h" +#include "oops/base/Model.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" #include "oops/base/Observers.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" +#include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" +#include "oops/util/algorithms.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" namespace oops { +// ----------------------------------------------------------------------------- + +/// \brief Options controlling the processing of observations from a single obs space. +template +class ObsTypeParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsTypeParameters, Parameters) + + public: + typedef typename ObsAuxControl::Parameters_ ObsAuxControlParameters_; + typedef ObsErrorParametersWrapper ObsErrorParameters_; + typedef typename ObsSpace::Parameters_ ObsSpaceParameters_; + + /// Options used to configure the observation space. + oops::RequiredParameter obsSpace{"obs space", this}; + + /// Options used to configure the observation operator, observation filters and GetValues. + ObserverParameters observer{this}; + + /// Options used to configure the observation error covariance matrix model. + oops::Parameter obsError{"obs error", {}, this}; + + /// Options used to configure bias correction. + oops::Parameter obsBias{"obs bias", {}, this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Top-level options taken by the HofX4D application. +template +class HofX4DParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(HofX4DParameters, Parameters) + + public: + typedef typename Geometry::Parameters_ GeometryParameters_; + typedef ModelParametersWrapper ModelParameters_; + + /// Only observations taken at times lying in the (`window begin`, `window begin` + `window + /// length`] interval will be included in observation spaces. + oops::RequiredParameter windowBegin{"window begin", this}; + oops::RequiredParameter windowLength{"window length", this}; + + /// Forecast length. + oops::RequiredParameter forecastLength{"forecast length", this}; + + /// A list whose elements determine treatment of observations from individual observation spaces. + oops::Parameter>> observations{"observations", {}, this}; + + /// Geometry parameters. + oops::RequiredParameter geometry{"geometry", this}; + + /// Model parameters. + oops::RequiredParameter model{"model", this}; + + /// Initial state parameters. + oops::RequiredParameter initialCondition{"initial condition", this}; + + /// Options passed to the object writing out forecast fields. + oops::Parameter prints{"prints", eckit::LocalConfiguration(), this}; + + /// Whether to perturb the H(x) vector before saving. + oops::Parameter obsPerturbations{"obs perturbations", false, this}; + + /// Whether to save the H(x) vector as ObsValues. + oops::Parameter makeObs{"make obs", false, this}; + + /// Parameters used by regression tests comparing results produced by the application against + /// known good outputs. + oops::Parameter test{"test", eckit::LocalConfiguration(), this}; +}; + +// ----------------------------------------------------------------------------- + /// Application runs model forecast from "initial condition" for the "forecast length" /// and computes H(x) on the run. If "obspert" is specified in the config, the resulting /// H(x) is perturbed. It is saved as "hofx" by default, or as specified "hofx group name" @@ -51,35 +130,44 @@ template class HofX4D : public Application { typedef ObsSpaces ObsSpaces_; typedef State State_; + typedef HofX4DParameters HofX4DParameters_; + typedef typename ObsAuxControl::Parameters_ ObsAuxControlParameters_; + typedef ObsErrorParametersWrapper ObsErrorParameters_; + typedef ObserverParameters ObserverParameters_; + typedef typename ObsSpace::Parameters_ ObsSpaceParameters_; + typedef ObsTypeParameters ObsTypeParameters_; + public: // ----------------------------------------------------------------------------- explicit HofX4D(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { instantiateObsErrorFactory(); instantiateObsFilterFactory(); + instantiateVariableChangeFactory(); } // ----------------------------------------------------------------------------- virtual ~HofX4D() = default; // ----------------------------------------------------------------------------- int execute(const eckit::Configuration & fullConfig) const { +// Deserialize parameters + HofX4DParameters_ params; + params.validateAndDeserialize(fullConfig); + // Setup observation window - const util::Duration winlen(fullConfig.getString("window length")); - const util::DateTime winbgn(fullConfig.getString("window begin")); + const util::Duration &winlen = params.windowLength; + const util::DateTime &winbgn = params.windowBegin; const util::DateTime winend(winbgn + winlen); Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; // Setup geometry - const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); - const Geometry_ geometry(geometryConfig, this->getComm()); + const Geometry_ geometry(params.geometry, this->getComm(), oops::mpi::myself()); // Setup Model - const eckit::LocalConfiguration modelConfig(fullConfig, "model"); - const Model_ model(geometry, modelConfig); + const Model_ model(geometry, params.model.value().modelParameters); // Setup initial state - const eckit::LocalConfiguration initialConfig(fullConfig, "initial condition"); - State_ xx(geometry, initialConfig); - ModelAux_ moderr(geometry, initialConfig); - const util::Duration flength(fullConfig.getString("forecast length")); + State_ xx(geometry, params.initialCondition); + ModelAux_ moderr(geometry, params.initialCondition); + const util::Duration &flength = params.forecastLength; Log::test() << "Initial state: " << xx << std::endl; // Check that window specified for forecast is at least the same as obs window @@ -93,18 +181,31 @@ template class HofX4D : public Application { // Setup forecast outputs PostProcessor post; - eckit::LocalConfiguration prtConf; - fullConfig.get("prints", prtConf); - post.enrollProcessor(new StateInfo("fc", prtConf)); + post.enrollProcessor(new StateInfo("fc", params.prints)); // Setup observations + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); - ObsAux_ obsaux(obspaces, obsConfig); - ObsErrors_ Rmat(obsConfig, obspaces); + const std::vector obsspaceParams = util::transformVector( + params.observations.value(), + [](const ObsTypeParameters_ & obsTypeParams) { return obsTypeParams.obsSpace.value(); }); + ObsSpaces_ obspaces(obsspaceParams, this->getComm(), winbgn, winend); + + const std::vector obsauxParams = util::transformVector( + params.observations.value(), + [](const ObsTypeParameters_ & obsTypeParams) { return obsTypeParams.obsBias.value(); }); + ObsAux_ obsaux(obspaces, obsauxParams); + + const std::vector obsErrorParams = util::transformVector( + params.observations.value(), + [](const ObsTypeParameters_ & obsTypeParams) { return obsTypeParams.obsError.value(); }); + ObsErrors_ Rmat(obsErrorParams, obspaces); // Setup and initialize observer - Observers_ hofx(obspaces, obsConfig); + std::vector observerParams = util::transformVector( + params.observations.value(), + [](const ObsTypeParameters_ & obsTypeParams) { return obsTypeParams.observer; }); + Observers_ hofx(obspaces, observerParams); hofx.initialize(geometry, obsaux, Rmat, post); // run the model and compute H(x) @@ -118,15 +219,13 @@ template class HofX4D : public Application { // Perturb H(x) if needed (can be used for generating obs in OSSE: perturbed H(x) could be saved // as ObsValue if "hofx group name" == ObsValue. - bool obspert = fullConfig.getBool("obs perturbations", false); - if (obspert) { + if (params.obsPerturbations) { yobs.perturb(Rmat); Log::test() << "Perturbed H(x): " << std::endl << yobs << "End Perturbed H(x)" << std::endl; } // Save H(x) as observations (if "make obs" == true) - const bool makeobs = fullConfig.getBool("make obs", false); - if (makeobs) yobs.save("ObsValue"); + if (params.makeObs) yobs.save("ObsValue"); obspaces.save(); return 0; diff --git a/src/oops/runs/HofX4Dhack.h b/src/oops/runs/HofX4Dhack.h index 335c7ac72..ebc5fea67 100644 --- a/src/oops/runs/HofX4Dhack.h +++ b/src/oops/runs/HofX4Dhack.h @@ -17,17 +17,17 @@ #include "eckit/exception/Exceptions.h" #include "oops/assimilation/CalcHofX.h" +#include "oops/base/Geometry.h" #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/ChangeVariables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/ConfigFunctions.h" @@ -163,7 +163,8 @@ template class HofX4Dhack : public Application { // loop over all observation types for (size_t jj = 0; jj < obspaces.size(); ++jj) { GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); - geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); + geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj], + geometry.variableSizes(vars[jj]))); getvals.fillGeoVaLs(zz, subWinBegin, subWinEnd, *geovals[jj]); } diff --git a/src/oops/runs/HybridGain.h b/src/oops/runs/HybridGain.h index 87ee0379b..5af101cda 100644 --- a/src/oops/runs/HybridGain.h +++ b/src/oops/runs/HybridGain.h @@ -14,10 +14,10 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/abor1_cpp.h" diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index baf6c50a3..76453d306 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -16,17 +16,18 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/instantiateLocalEnsembleSolverFactory.h" #include "oops/assimilation/LocalEnsembleSolver.h" -#include "oops/assimilation/State4D.h" #include "oops/base/Departures.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State4D.h" #include "oops/base/StateEnsemble4D.h" #include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/interface/Geometry.h" +#include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/Increment.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -56,6 +57,7 @@ template class LocalEnsembleDA : public Applicati instantiateLocalEnsembleSolverFactory(); instantiateObsErrorFactory(); instantiateObsFilterFactory(); + instantiateVariableChangeFactory(); } // ----------------------------------------------------------------------------- @@ -93,14 +95,16 @@ template class LocalEnsembleDA : public Applicati // Get background configurations const eckit::LocalConfiguration bgConfig(fullConfig, "background"); - // Read all ensemble members + // Read all ensemble members and compute the mean StateEnsemble4D_ ens_xx(geometry, bgConfig); const size_t nens = ens_xx.size(); const Variables statevars = ens_xx.variables(); + State4D_ bkg_mean = ens_xx.mean(); // set up solver std::unique_ptr solver = - LocalEnsembleSolverFactory::create(obsdb, geometry, fullConfig, nens); + LocalEnsembleSolverFactory::create(obsdb, geometry, fullConfig, + nens, bkg_mean); // test prints for the prior ensemble bool do_test_prints = driverConfig.getBool("do test prints", true); if (do_test_prints) { @@ -125,8 +129,7 @@ template class LocalEnsembleDA : public Applicati return 0; } - // calculate background mean - State4D_ bkg_mean = ens_xx.mean(); + // print background mean if (do_test_prints) { Log::test() << "Background mean :" << bkg_mean << std::endl; } @@ -144,6 +147,10 @@ template class LocalEnsembleDA : public Applicati } Log::info() << "Local solver completed." << std::endl; + // wait all tasks to finish their solution, so the timing for functions below reports + // time which truly used (not from mpi_wait(), as all tasks need to sync before write). + oops::mpi::world().barrier(); + // calculate final analysis states for (size_t jj = 0; jj < nens; ++jj) { ens_xx[jj] = bkg_mean; diff --git a/src/oops/runs/RTPP.h b/src/oops/runs/RTPP.h index 401068cf4..4ae340ee6 100644 --- a/src/oops/runs/RTPP.h +++ b/src/oops/runs/RTPP.h @@ -12,12 +12,10 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/Increment4D.h" -#include "oops/assimilation/State4D.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/StateEnsemble.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/Logger.h" diff --git a/src/oops/runs/StaticBInit.h b/src/oops/runs/StaticBInit.h index d36ed2a8f..f8d3cd415 100644 --- a/src/oops/runs/StaticBInit.h +++ b/src/oops/runs/StaticBInit.h @@ -12,11 +12,11 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/Geometry.h" #include "oops/base/instantiateCovarFactory.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/Logger.h" diff --git a/src/oops/runs/Variational.h b/src/oops/runs/Variational.h index c57968443..7cf4d1a31 100644 --- a/src/oops/runs/Variational.h +++ b/src/oops/runs/Variational.h @@ -23,28 +23,22 @@ #include "oops/assimilation/instantiateMinFactory.h" #include "oops/base/instantiateCovarFactory.h" #include "oops/base/instantiateObsFilterFactory.h" -#include "oops/base/Observations.h" #include "oops/base/PostProcessor.h" +#include "oops/base/State.h" #include "oops/base/StateInfo.h" #include "oops/base/StateWriter.h" +#include "oops/generic/instantiateLinearModelFactory.h" #include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/generic/instantiateTlmFactory.h" #include "oops/generic/instantiateVariableChangeFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" -#include "oops/util/Duration.h" #include "oops/util/Logger.h" #include "oops/util/printRunStats.h" namespace oops { template class Variational : public Application { - typedef Geometry Geometry_; - typedef Model Model_; typedef State State_; public: @@ -55,7 +49,7 @@ template class Variational : public Application { instantiateMinFactory(); instantiateObsErrorFactory(); instantiateObsFilterFactory(); - instantiateTlmFactory(); + instantiateLinearModelFactory(); instantiateVariableChangeFactory(); } // ----------------------------------------------------------------------------- diff --git a/src/oops/util/TestReference.cc b/src/oops/util/TestReference.cc index 08f3e435b..231e72242 100644 --- a/src/oops/util/TestReference.cc +++ b/src/oops/util/TestReference.cc @@ -108,11 +108,18 @@ void TestReference::compare(const std::string & test, while (std::getline(refStream, refLine)) { lineCounter++; - std::getline(testStream, testLine); - // Check that a corresponding test line exists - if (testStream.fail()) { - throw TestReferenceMissingTestLineError(lineCounter, refLine); + // If reference line is empty, go to the next line + if (refLine.empty()) continue; + + testLine = ""; + // Find non-empty line in the test output + while (testLine.empty()) { + std::getline(testStream, testLine); + // Check that a corresponding test line exists + if (testStream.fail()) { + throw TestReferenceMissingTestLineError(lineCounter, refLine); + } } // Remove test prefix from refLine if present @@ -152,8 +159,12 @@ void TestReference::compare(const std::string & test, } // Check there are no more remaining test lines to process - std::getline(testStream, testLine); - if (!testStream.fail()) { + testLine = ""; + // Read until either reach end of file or find non-empty line in the test output + while (testLine.empty() && (!testStream.fail())) { + std::getline(testStream, testLine); + } + if (!testLine.empty()) { throw TestReferenceMissingReferenceLineError(lineCounter, testLine); } } diff --git a/src/oops/util/TimerHelper.cc b/src/oops/util/TimerHelper.cc index 729e40ce6..70ce12a9c 100644 --- a/src/oops/util/TimerHelper.cc +++ b/src/oops/util/TimerHelper.cc @@ -14,7 +14,7 @@ #include #include -#include "eckit/io/ResizableBuffer.h" +#include "eckit/io/Buffer.h" #include "eckit/mpi/Comm.h" #include "eckit/serialisation/ResizableMemoryStream.h" @@ -101,7 +101,7 @@ void TimerHelper::print(std::ostream & os) const { int tag = 1234; // Tasks send their numbers to task 0 if (oops::mpi::world().rank() > 0) { - eckit::ResizableBuffer bufr(8000); + eckit::Buffer bufr(8000); eckit::ResizableMemoryStream sstr(bufr); sstr << timers_.size(); for (cit jt = timers_.begin(); jt != timers_.end(); ++jt) { @@ -118,7 +118,7 @@ void TimerHelper::print(std::ostream & os) const { for (size_t from = 1; from < ntasks; ++from) { eckit::mpi::Status st = oops::mpi::world().probe(from, tag); size_t size = oops::mpi::world().getCount(st); - eckit::ResizableBuffer bufr(size); + eckit::Buffer bufr(size); bufr.zero(); oops::mpi::world().receive(static_cast(bufr.data()), bufr.size(), from, tag); diff --git a/src/oops/util/algorithms.h b/src/oops/util/algorithms.h new file mode 100644 index 000000000..05f2da2ff --- /dev/null +++ b/src/oops/util/algorithms.h @@ -0,0 +1,32 @@ +/* + * (C) Crown copyright 2021 Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_ALGORITHMS_H_ +#define OOPS_UTIL_ALGORITHMS_H_ + +#include +#include + +namespace util { + +/// \brief Returns the vector containing the results of calling the function `op` on each element +/// of the vector `inputs`. +template +std::vector::type> transformVector( + const std::vector &inputs, const UnaryOperation &op) +{ + typedef typename std::result_of::type Output; + std::vector outputs; + outputs.reserve(inputs.size()); + for (const Input &input : inputs) + outputs.push_back(op(input)); + return outputs; +} + +} // namespace util + +#endif // OOPS_UTIL_ALGORITHMS_H_ diff --git a/src/oops/util/missing_values_f.cc b/src/oops/util/missing_values_f.cc index ba2885118..f14a7d600 100644 --- a/src/oops/util/missing_values_f.cc +++ b/src/oops/util/missing_values_f.cc @@ -13,22 +13,22 @@ namespace util { // ----------------------------------------------------------------------------- int32_t missing_value_int32_f() { - const int32_t miss = util::missingValue(miss); + const int32_t miss = util::missingValue(int32_t()); return miss; } // ----------------------------------------------------------------------------- int64_t missing_value_int64_f() { - const int64_t miss = util::missingValue(miss); + const int64_t miss = util::missingValue(int64_t()); return miss; } // ----------------------------------------------------------------------------- float missing_value_flt_f() { - const float miss = util::missingValue(miss); + const float miss = util::missingValue(float()); return miss; } // ----------------------------------------------------------------------------- double missing_value_dbl_f() { - const double miss = util::missingValue(miss); + const double miss = util::missingValue(double()); return miss; } // ----------------------------------------------------------------------------- diff --git a/src/oops/util/parameters/OptionalParameter.cc b/src/oops/util/parameters/OptionalParameter.cc index d133ef0d5..320d61c32 100644 --- a/src/oops/util/parameters/OptionalParameter.cc +++ b/src/oops/util/parameters/OptionalParameter.cc @@ -52,7 +52,11 @@ void OptionalParameter::serialize(eckit::LocalConfiguration &config) const } ObjectJsonSchema OptionalParameter::jsonSchema() const { - return ObjectJsonSchema({{name_, {{"type", "\"null\""}}}}); + ObjectJsonSchema schema = ObjectJsonSchema({{name_, {{"type", "\"null\""}}}}); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } + return schema; } } // namespace oops diff --git a/src/oops/util/parameters/OptionalParameter.h b/src/oops/util/parameters/OptionalParameter.h index 7824f12d6..46aa0da80 100644 --- a/src/oops/util/parameters/OptionalParameter.h +++ b/src/oops/util/parameters/OptionalParameter.h @@ -53,7 +53,31 @@ class OptionalParameter : public ParameterBase { explicit OptionalParameter( const char *name, Parameters *parent, std::vector>> constraints = {}) - : ParameterBase(parent), name_(name), constraints_(std::move(constraints)) + : OptionalParameter(name, "", parent, constraints) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the key from which this parameter's value will be loaded when parameters are + /// deserialized from a Configuration object. Similarly, name of the key to which this + /// parameter's value will be saved when parameters are serialized to a Configuration object. + /// \param description + /// Long description of this parameter. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + /// \param constraints + /// Zero or more constraints that must be satisfied by the value of this parameter loaded from + /// a Configuration object; if that's not the case, an exception will be thrown during + /// deserialization. + explicit OptionalParameter( + const char *name, const char *description, Parameters *parent, + std::vector>> constraints = {}) + : ParameterBase(parent), name_(name), description_(description), + constraints_(std::move(constraints)) {} void deserialize(util::CompositePath &path, const eckit::Configuration &config) override; @@ -70,6 +94,7 @@ class OptionalParameter : public ParameterBase { private: std::string name_; + std::string description_; boost::optional value_; std::vector>> constraints_; }; @@ -95,6 +120,9 @@ void OptionalParameter::serialize(eckit::LocalConfiguration &config) const { template ObjectJsonSchema OptionalParameter::jsonSchema() const { ObjectJsonSchema schema = ParameterTraits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } for (const std::shared_ptr> &constraint : constraints_) { PropertyJsonSchema constraintSchema = constraint->jsonSchema(); schema.extendPropertySchema(name_, std::move(constraintSchema)); @@ -134,9 +162,26 @@ class OptionalParameter : public ParameterBase { /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() /// on that object will automatically trigger a call to deserialize() or serialize() on this /// parameter. + OptionalParameter(const char *name, Parameters *parent) + : OptionalParameter(name, "", parent) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the key from which this parameter's value will be loaded when parameters are + /// deserialized from a Configuration object. Similarly, name of the key to which this + /// parameter's value will be saved when parameters are serialized to a Configuration object. + /// \param description + /// Long description of this parameter. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. OptionalParameter( - const char *name, Parameters *parent) - : ParameterBase(parent), name_(name), value_(false) + const char *name, const char *description, Parameters *parent) + : ParameterBase(parent), name_(name), description_(description), value_(false) {} void deserialize(util::CompositePath &path, const eckit::Configuration &config) override; @@ -155,6 +200,7 @@ class OptionalParameter : public ParameterBase { private: std::string name_; + std::string description_; bool value_; }; diff --git a/src/oops/util/parameters/OptionalPolymorphicParameter.h b/src/oops/util/parameters/OptionalPolymorphicParameter.h index 25ed83b6a..fa1d2e413 100644 --- a/src/oops/util/parameters/OptionalPolymorphicParameter.h +++ b/src/oops/util/parameters/OptionalPolymorphicParameter.h @@ -49,11 +49,29 @@ class OptionalPolymorphicParameter : public ParameterBase { /// on that object will automatically trigger a call to deserialize() or serialize() on this /// parameter. explicit OptionalPolymorphicParameter(const char *name, Parameters *parent) - : ParameterBase(parent), name_(name) + : OptionalPolymorphicParameter(name, "", parent) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the configuration key whose value determines the concrete subclass of `PARAMETERS` + /// created during deserialization. + /// \param description + /// Long description of this parameter. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as `name`. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + explicit OptionalPolymorphicParameter(const char *name, const char * description, + Parameters *parent) + : ParameterBase(parent), name_(name), description_(description) {} OptionalPolymorphicParameter(const OptionalPolymorphicParameter &other) - : name_(other.name_), id_(other.id_), value_(other.value_ ? other.value_->clone() : nullptr) + : name_(other.name_), description_(other.description_), id_(other.id_), + value_(other.value_ ? other.value_->clone() : nullptr) {} OptionalPolymorphicParameter(OptionalPolymorphicParameter &&other) = default; @@ -88,6 +106,7 @@ class OptionalPolymorphicParameter : public ParameterBase { typedef PolymorphicParameterTraits Traits; std::string name_; + std::string description_; boost::optional id_; std::unique_ptr value_; }; @@ -113,7 +132,11 @@ void OptionalPolymorphicParameter::serialize( template ObjectJsonSchema OptionalPolymorphicParameter::jsonSchema() const { - return Traits::jsonSchema(name_); + ObjectJsonSchema schema = Traits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } + return schema; } } // namespace oops diff --git a/src/oops/util/parameters/Parameter.h b/src/oops/util/parameters/Parameter.h index f924db82f..b849dc638 100644 --- a/src/oops/util/parameters/Parameter.h +++ b/src/oops/util/parameters/Parameter.h @@ -58,7 +58,33 @@ class Parameter : public ParameterBase { /// deserialization. Parameter(const char *name, const T& defaultValue, Parameters *parent, std::vector>> constraints = {}) - : ParameterBase(parent), name_(name), value_(defaultValue), constraints_(std::move(constraints)) + : Parameter(name, "", defaultValue, parent, constraints) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the key from which this parameter's value will be loaded when parameters are + /// deserialized from a Configuration object. Similarly, name of the key to which this + /// parameter's value will be saved when parameters are serialized to a Configuration object. + /// \param description + /// Long description of this parameter. + /// \param defaultValue + /// Default value of the parameter, used if the Configuration object from which parameters are + /// deserialized doesn't contain a key with name \p name. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + /// \param constraints + /// Zero or more constraints that must be satisfied by the value of this parameter loaded from + /// a Configuration object; if that's not the case, an exception will be thrown during + /// deserialization. + Parameter(const char *name, const char *description, const T& defaultValue, Parameters *parent, + std::vector>> constraints = {}) + : ParameterBase(parent), name_(name), description_(description), value_(defaultValue), + constraints_(std::move(constraints)) {} void deserialize(util::CompositePath &path, const eckit::Configuration &config) override; @@ -75,6 +101,7 @@ class Parameter : public ParameterBase { private: std::string name_; + std::string description_; T value_; std::vector>> constraints_; }; @@ -99,6 +126,9 @@ void Parameter::serialize(eckit::LocalConfiguration &config) const { template ObjectJsonSchema Parameter::jsonSchema() const { ObjectJsonSchema schema = ParameterTraits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } for (const std::shared_ptr> &constraint : constraints_) { PropertyJsonSchema constraintSchema = constraint->jsonSchema(); schema.extendPropertySchema(name_, std::move(constraintSchema)); diff --git a/src/oops/util/parameters/Parameters.h b/src/oops/util/parameters/Parameters.h index 5d1830494..7b9958f41 100644 --- a/src/oops/util/parameters/Parameters.h +++ b/src/oops/util/parameters/Parameters.h @@ -211,6 +211,22 @@ ParametersType validateAndDeserialize(const eckit::Configuration &config) { return parameters; } +/// \brief Deserialize configurations \p configs into new instances of \c ParametersType +/// after validating them against the JSON schema defined by \c ParametersType. +/// +/// If \p configs are not top-level configurations loaded from YAML files, then paths to nodes +/// of the YAML tree included in any error messages will be incomplete. +/// +/// \tparam ParametersType A concrete subclass of Parameters. +template +std::vector validateAndDeserialize( + const std::vector &configs) { + std::vector parameters(configs.size()); + for (size_t i = 0; i < configs.size(); ++i) + parameters[i].validateAndDeserialize(configs[i]); + return parameters; +} + } // namespace oops #endif // OOPS_UTIL_PARAMETERS_PARAMETERS_H_ diff --git a/src/oops/util/parameters/PolymorphicParameter.h b/src/oops/util/parameters/PolymorphicParameter.h index a69847d4a..fff83f654 100644 --- a/src/oops/util/parameters/PolymorphicParameter.h +++ b/src/oops/util/parameters/PolymorphicParameter.h @@ -49,10 +49,30 @@ class PolymorphicParameter : public ParameterBase { /// Pointer to the Parameters object representing the collection of options located at /// the same level of the configuration tree as `name`. A call to deserialize() on that object /// will automatically trigger a call to deserialize() on the newly created object. - PolymorphicParameter(const char *name, const char* defaultId, Parameters *parent); + PolymorphicParameter(const char *name, const char* defaultId, Parameters *parent) + : PolymorphicParameter(name, "", defaultId, parent) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the configuration key whose value determines the concrete subclass of `PARAMETERS` + /// created during deserialization. + /// \param description + /// Long description of this parameter. + /// \param defaultId + /// Identifier of the concrete subclass of `PARAMETERS` to be created if the above key is not + /// present. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as `name`. A call to deserialize() on that object + /// will automatically trigger a call to deserialize() on the newly created object. + PolymorphicParameter(const char *name, const char *description, const char* defaultId, + Parameters *parent); PolymorphicParameter(const PolymorphicParameter &other) - : name_(other.name_), id_(other.id_), value_(other.value_ ? other.value_->clone() : nullptr) + : name_(other.name_), description_(other.description_), id_(other.id_), + value_(other.value_ ? other.value_->clone() : nullptr) {} PolymorphicParameter(PolymorphicParameter &&other) = default; @@ -86,15 +106,17 @@ class PolymorphicParameter : public ParameterBase { typedef PolymorphicParameterTraits Traits; std::string name_; + std::string description_; std::string id_; std::unique_ptr value_; }; template PolymorphicParameter::PolymorphicParameter(const char *name, + const char *description, const char* defaultId, Parameters *parent) - : ParameterBase(parent), name_(name) { + : ParameterBase(parent), name_(name), description_(description) { util::CompositePath path; eckit::LocalConfiguration config; config.set(name, defaultId); @@ -120,7 +142,11 @@ void PolymorphicParameter::serialize(eckit::LocalConfigurat template ObjectJsonSchema PolymorphicParameter::jsonSchema() const { - return Traits::jsonSchema(name_); + ObjectJsonSchema schema = Traits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } + return schema; } } // namespace oops diff --git a/src/oops/util/parameters/RequiredParameter.h b/src/oops/util/parameters/RequiredParameter.h index cb351e9d3..27523605f 100644 --- a/src/oops/util/parameters/RequiredParameter.h +++ b/src/oops/util/parameters/RequiredParameter.h @@ -53,7 +53,31 @@ class RequiredParameter : public ParameterBase { explicit RequiredParameter( const char *name, Parameters *parent, std::vector>> constraints = {}) - : ParameterBase(parent), name_(name), constraints_(std::move(constraints)) + : RequiredParameter(name, "", parent, constraints) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the key from which this parameter's value will be loaded when parameters are + /// deserialized from a Configuration object. Similarly, name of the key to which this + /// parameter's value will be saved when parameters are serialized to a Configuration object. + /// \param description + /// Long description of this parameter. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + /// \param constraints + /// Zero or more constraints that must be satisfied by the value of this parameter loaded from + /// a Configuration object; if that's not the case, an exception will be thrown during + /// deserialization. + explicit RequiredParameter( + const char *name, const char *description, Parameters *parent, + std::vector>> constraints = {}) + : ParameterBase(parent), name_(name), description_(description), + constraints_(std::move(constraints)) {} /// \brief Load the value of this parameter from \p config. @@ -78,6 +102,7 @@ class RequiredParameter : public ParameterBase { private: std::string name_; + std::string description_; // The value is stored in a boost::optional object because T may not be // default-constructible. boost::optional value_; @@ -106,6 +131,9 @@ void RequiredParameter::serialize(eckit::LocalConfiguration &config) const { template ObjectJsonSchema RequiredParameter::jsonSchema() const { ObjectJsonSchema schema = ParameterTraits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } schema.require(name_); for (const std::shared_ptr> &constraint : constraints_) { PropertyJsonSchema constraintSchema = constraint->jsonSchema(); diff --git a/src/oops/util/parameters/RequiredPolymorphicParameter.h b/src/oops/util/parameters/RequiredPolymorphicParameter.h index 3c27092f0..1cc844177 100644 --- a/src/oops/util/parameters/RequiredPolymorphicParameter.h +++ b/src/oops/util/parameters/RequiredPolymorphicParameter.h @@ -106,11 +106,29 @@ class RequiredPolymorphicParameter : public ParameterBase { /// on that object will automatically trigger a call to deserialize() or serialize() on this /// parameter. explicit RequiredPolymorphicParameter(const char *name, Parameters *parent) - : ParameterBase(parent), name_(name) + : RequiredPolymorphicParameter(name, "", parent) + {} + + /// \brief Constructor. + /// + /// \param name + /// Name of the configuration key whose value determines the concrete subclass of `PARAMETERS` + /// created during deserialization. + /// \param description + /// Long description of this parameter. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as `name`. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + explicit RequiredPolymorphicParameter(const char *name, const char *description, + Parameters *parent) + : ParameterBase(parent), name_(name), description_(description) {} RequiredPolymorphicParameter(const RequiredPolymorphicParameter &other) - : name_(other.name_), id_(other.id_), value_(other.value_ ? other.value_->clone() : nullptr) + : name_(other.name_), description_(other.description_), id_(other.id_), + value_(other.value_ ? other.value_->clone() : nullptr) {} RequiredPolymorphicParameter(RequiredPolymorphicParameter &&other) = default; @@ -159,6 +177,7 @@ class RequiredPolymorphicParameter : public ParameterBase { typedef PolymorphicParameterTraits Traits; std::string name_; + std::string description_; std::string id_; std::unique_ptr value_; }; @@ -186,6 +205,9 @@ void RequiredPolymorphicParameter::serialize( template ObjectJsonSchema RequiredPolymorphicParameter::jsonSchema() const { ObjectJsonSchema schema = Traits::jsonSchema(name_); + if (description_ != "") { + schema.extendPropertySchema(name_, {{"description", "\"" + description_ + "\""}}); + } schema.require(name_); return schema; } diff --git a/src/oops/util/signal_trap.cc b/src/oops/util/signal_trap.cc index 240361592..ff7d725db 100644 --- a/src/oops/util/signal_trap.cc +++ b/src/oops/util/signal_trap.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019 UCAR + * (C) Copyright 2019-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -12,13 +12,18 @@ #include // feenableexcept #endif -#include // backtrace* #include // strerror(errno) #include // sigaction, siginfo_t -#include // abort #include // strerror #include // cout, cerr -#include "oops/util/Logger.h" // required for oops::Log + +// The behavior of boost::stacktrace is controlled by preprocessor macros. See +// https://www.boost.org/doc/libs/1_76_0/doc/html/stacktrace/configuration_and_build.html +// for details. The correct libs and macros are set using CMake (in backtrace_deps.cmake). +#include // boost stacktraces + +#include "oops/util/abor1_cpp.h" // ABORT macro +#include "oops/util/Logger.h" // required for oops::Log void trap_sigfpe(const int); // user function traps SIGFPE void sigfpe_handler(int, siginfo_t *, void *); // called when relevant SIGFPE occurs @@ -62,12 +67,6 @@ void trap_sigfpe(const int abortflg) // This is the signal handler invoked when SIGFPE encountered // Arguments "sig" and "ucontext" are required but unused here void sigfpe_handler(int sig, siginfo_t *info, void *ucontext) { - static const int maxfuncs = 50; // gather no more than this many functions in the backtrace - void *stack[maxfuncs]; // call stack - char **stacknames; // backtrace function names - size_t nfuncs; // number of functions returned by backtrace - size_t n; // iterator over nfuncs - // myrank = eckit::mpi::comm().rank(); LOGIT_STDERR << "Caught SIGFPE: "; @@ -96,12 +95,8 @@ void sigfpe_handler(int sig, siginfo_t *info, void *ucontext) { } if (do_abort) { - nfuncs = backtrace(stack, maxfuncs); // number of functions in the backtrace - stacknames = backtrace_symbols(&stack[0], nfuncs); // generate the backtrace names from symbols - // Loop through the callstack, printing the backtrace - for (n = 0; n < nfuncs; ++n) { - LOGIT_STDERR << stacknames[n] << std::endl; - } - abort(); // exit + LOGIT_STDERR << boost::stacktrace::stacktrace() << std::endl; + ABORT("A SIGFPE was encountered. If available, see frame #2 in the stacktrace for details."); } } + diff --git a/src/test/base/ObsErrorCovariance.h b/src/test/base/ObsErrorCovariance.h index ae4f68d47..4c864da6e 100644 --- a/src/test/base/ObsErrorCovariance.h +++ b/src/test/base/ObsErrorCovariance.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -18,9 +18,9 @@ #define ECKIT_TESTING_SELF_REGISTER_CASES 0 #include "eckit/testing/Test.h" -#include "oops/base/ObsErrorBase.h" +#include "oops/base/ObsError.h" +#include "oops/base/ObsVector.h" #include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" @@ -30,9 +30,9 @@ namespace test { // ----------------------------------------------------------------------------- /// Tests creation and destruction of ObsErrorCovariances template void testConstructor() { - typedef ObsTestsFixture Test_; - typedef oops::ObsErrorBase Covar_; - typedef oops::ObsVector ObsVector_; + typedef ObsTestsFixture Test_; + typedef oops::ObsError Covar_; + typedef oops::ObsErrorParametersWrapper Parameters_; oops::instantiateObsErrorFactory(); @@ -40,12 +40,11 @@ template void testConstructor() { TestEnvironment::config().get("observations", conf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - ObsVector_ obserr(Test_::obspace()[jj], "ObsError"); - obserr.save("EffectiveError"); - const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - std::unique_ptr R( - oops::ObsErrorFactory::create(rconf, Test_::obspace()[jj])); + Parameters_ rparams; + rparams.validateAndDeserialize(rconf); + std::unique_ptr R = std::make_unique(rparams.obsErrorParameters, + Test_::obspace()[jj]); EXPECT(R.get()); oops::Log::test() << "Testing ObsError: " << *R << std::endl; R.reset(); @@ -56,9 +55,10 @@ template void testConstructor() { // ----------------------------------------------------------------------------- /// Tests that \f$R*R^{-1}*dy = dy\f$ and \f$R^{-1}*R*dy = dy\f$ template void testMultiplies() { - typedef ObsTestsFixture Test_; - typedef oops::ObsErrorBase Covar_; - typedef oops::ObsVector ObsVector_; + typedef ObsTestsFixture Test_; + typedef oops::ObsError Covar_; + typedef oops::ObsErrorParametersWrapper Parameters_; + typedef oops::ObsVector ObsVector_; oops::instantiateObsErrorFactory(); @@ -67,36 +67,81 @@ template void testMultiplies() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { ObsVector_ obserr(Test_::obspace()[jj], "ObsError"); - obserr.save("EffectiveError"); const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - std::unique_ptr R( - oops::ObsErrorFactory::create(rconf, Test_::obspace()[jj])); + Parameters_ rparams; + rparams.validateAndDeserialize(rconf); + Covar_ R(rparams.obsErrorParameters, Test_::obspace()[jj]); // RMSE should be equal to the rms that was read from the file - EXPECT(oops::is_close(R->getRMSE(), obserr.rms(), 1.e-10)); + EXPECT(oops::is_close(R.getRMSE(), obserr.rms(), 1.e-10)); // create random vector dy and its copies dy1, dy2 ObsVector_ dy(Test_::obspace()[jj]); - dy.random(); + R.randomize(dy); ObsVector_ dy1(dy); ObsVector_ dy2(dy); - oops::Log::info() << "Random vector dy: " << dy << std::endl; + oops::Log::test() << "Random vector dy: " << dy << std::endl; - R->multiply(dy1); - R->inverseMultiply(dy1); + R.multiply(dy1); + oops::Log::test() << "R*dy: " << dy1 << std::endl; + R.inverseMultiply(dy1); // dy1 = R^{-1}*R*dy - oops::Log::info() << "R^{-1}*R*dy: " << dy1 << std::endl; + oops::Log::test() << "R^{-1}*R*dy: " << dy1 << std::endl; EXPECT(oops::is_close(dy1.rms(), dy.rms(), 1.e-10)); - R->inverseMultiply(dy2); - R->multiply(dy2); + R.inverseMultiply(dy2); + oops::Log::test() << "R^{-1}*dy: " << dy2 << std::endl; + R.multiply(dy2); // dy2 = R*R^P-1}*dy - oops::Log::info() << "R*R^{-1}*dy: " << dy2 << std::endl; + oops::Log::test() << "R*R^{-1}*dy: " << dy2 << std::endl; EXPECT(oops::is_close(dy2.rms(), dy.rms(), 1.e-10)); } } +// ----------------------------------------------------------------------------- +/// Tests that the methods obserrors(), inverseVariance update() and save() +/// do what is expected. +template void testAccessors() { + typedef ObsTestsFixture Test_; + typedef oops::ObsError Covar_; + typedef oops::ObsErrorParametersWrapper Parameters_; + typedef oops::ObsVector ObsVector_; + + oops::instantiateObsErrorFactory(); + + std::vector conf; + TestEnvironment::config().get("observations", conf); + + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + ObsVector_ obserr(Test_::obspace()[jj], "ObsError"); + + const eckit::LocalConfiguration rconf(conf[jj], "obs error"); + Parameters_ rparams; + rparams.validateAndDeserialize(rconf); + Covar_ R(rparams.obsErrorParameters, Test_::obspace()[jj]); + + ObsVector_ dy(R.obserrors()); + oops::Log::test() << "ObsError: " << dy << std::endl; + EXPECT(oops::is_close(dy.rms(), obserr.rms(), 1.e-10)); + + ObsVector_ dy1(R.inverseVariance()); + oops::Log::test() << "inverseVariance: " << dy1 << std::endl; + dy *= dy; + dy.invert(); + EXPECT(oops::is_close(dy.rms(), dy1.rms(), 1.e-10)); + + dy.ones(); + R.update(dy); + oops::Log::test() << "R filled with ones: " << R.obserrors() << std::endl; + EXPECT(oops::is_close(R.obserrors().rms(), R.inverseVariance().rms(), 1.e-10)); + EXPECT(oops::is_close(R.obserrors().rms(), dy.rms(), 1.e-10)); + + R.save("Ones"); + ObsVector_ testOnes(Test_::obspace()[jj], "Ones"); + EXPECT(oops::is_close(dy.rms(), testOnes.rms(), 1.e-10)); + } +} // ----------------------------------------------------------------------------- @@ -116,6 +161,8 @@ class ObsErrorCovariance : public oops::Test { { testConstructor(); }); ts.emplace_back(CASE("interface/ObsErrorCovariance/testMultiplies") { testMultiplies(); }); + ts.emplace_back(CASE("interface/ObsErrorCovariance/testAccessors") + { testAccessors(); }); } void clear() const override { diff --git a/src/test/base/Variables.h b/src/test/base/Variables.h index f9fd2498b..ba84f665d 100644 --- a/src/test/base/Variables.h +++ b/src/test/base/Variables.h @@ -52,10 +52,18 @@ void testConstructor() { EXPECT(other->channels() == channels); } + { + const std::vector varnames{}; + const std::vector channels{}; + oops::Variables other(TestEnvironment::config(), "empty variables"); + EXPECT(other.variables() == varnames); + EXPECT(other.channels() == channels); + } + vars.reset(); EXPECT(!vars.get()); - // Test bad construction from eckit::Configuration + // Test construction from an eckit::Configuration entry that does not exist EXPECT_THROWS_AS(auto bad_vars = oops::Variables(TestEnvironment::config(), "FOO"), eckit::BadParameter); } diff --git a/src/test/generic/PseudoModelState4D.h b/src/test/generic/PseudoModelState4D.h new file mode 100644 index 000000000..316bd8aaf --- /dev/null +++ b/src/test/generic/PseudoModelState4D.h @@ -0,0 +1,105 @@ +/* + * (C) Copyright 2021- UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_GENERIC_PSEUDOMODELSTATE4D_H_ +#define TEST_GENERIC_PSEUDOMODELSTATE4D_H_ + +#include +#include +#include +#include + +#define ECKIT_TESTING_SELF_REGISTER_CASES 0 + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Model.h" +#include "oops/base/PostProcessor.h" +#include "oops/base/State4D.h" +#include "oops/generic/PseudoModelState4D.h" +#include "oops/interface/ModelAuxControl.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Test.h" +#include "oops/util/Duration.h" +#include "test/TestEnvironment.h" + +namespace test { + +/// \brief Tests that PseudoModelState4D, run to 0th, 1st, 2nd, etc +/// state produces the same results as the Model specified in yaml. +template void testPseudoModelState4D() { + typedef oops::Geometry Geometry_; + typedef oops::Model Model_; + typedef oops::ModelAuxControl ModelAux_; + typedef oops::State State_; + typedef oops::State4D State4D_; + typedef oops::ModelBase ModelBase_; + typedef oops::PseudoModelState4D PseudoModelState4D_; + + // Setup geometry, model bias, and initial conditions + const eckit::LocalConfiguration geometryconf(TestEnvironment::config(), "geometry"); + Geometry_ geometry(geometryconf, oops::mpi::world()); + const eckit::LocalConfiguration biasconf(TestEnvironment::config(), "model aux control"); + ModelAux_ modelaux(geometry, biasconf); + const eckit::LocalConfiguration initconf(TestEnvironment::config(), "initial condition"); + State_ xinit(geometry, initconf); + + // set up Model specified in yaml + const eckit::LocalConfiguration modelconf(TestEnvironment::config(), "model"); + Model_ model1(geometry, modelconf); + oops::Log::test() << "Model: " << model1 << std::endl; + + // set up PseudoModelState4D with State4D from yaml + const eckit::LocalConfiguration state4dconf(TestEnvironment::config(), + "pseudo model with 4D state"); + State4D_ state4d(geometry, state4dconf); + std::unique_ptr pseudomodel(new PseudoModelState4D_(state4d)); + Model_ model2(std::move(pseudomodel)); + oops::Log::test() << "PseudoModelState4D: " << model2 << std::endl; + + // check that the two models have the same time resolutions + const util::Duration step = model1.timeResolution(); + EXPECT(step == model2.timeResolution()); + const size_t nsteps = state4d.size(); + + // run forecasts with 0, step, 2*step, ... lengths and compare the results + for (size_t jstep = 0; jstep < nsteps; ++jstep) { + State_ x1(xinit); + State_ x2(xinit); + oops::PostProcessor post; + oops::Log::test() << "Running Model for " << jstep << " steps" << std::endl; + model1.forecast(x1, modelaux, step * jstep, post); + oops::Log::test() << "Running PseudoModelState4D for " << jstep << " steps" << std::endl; + model2.forecast(x2, modelaux, step * jstep, post); + EXPECT(x1.norm() == x2.norm()); + EXPECT(x1.norm() == state4d[jstep].norm()); + } +} + +template +class PseudoModelState4D : public oops::Test { + public: + PseudoModelState4D() = default; + virtual ~PseudoModelState4D() = default; + private: + std::string testid() const override {return "test::PseudoModelState4D<" + MODEL::name() + ">";} + + void register_tests() const override { + std::vector& ts = eckit::testing::specification(); + + ts.emplace_back(CASE("generic/PseudoModelState4D/testPseudoModelState4D") + { testPseudoModelState4D(); }); + } + + void clear() const override {} +}; + +} // namespace test + +#endif // TEST_GENERIC_PSEUDOMODELSTATE4D_H_ diff --git a/src/test/generic/VerticalLocEV.h b/src/test/generic/VerticalLocEV.h index 1b89bc459..87eeab6e5 100644 --- a/src/test/generic/VerticalLocEV.h +++ b/src/test/generic/VerticalLocEV.h @@ -23,11 +23,12 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" -#include "oops/assimilation/Increment4D.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/Increment4D.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/Variables.h" #include "oops/generic/VerticalLocEV.h" -#include "oops/interface/Geometry.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" @@ -44,10 +45,15 @@ template void testVerticalLocEV() { typedef oops::VerticalLocEV VerticalLocEV_; typedef oops::Increment4D Increment4D_; typedef oops::IncrementEnsemble4D IncrementEnsemble_; + typedef oops::State State_; const Geometry_ & geometry = Test_::resol(); eckit::LocalConfiguration vertlocconf(TestEnvironment::config(), "vertical localization"); - VerticalLocEV_ vertloc(geometry, vertlocconf); + + // make an empty state vector to be used to intitialize VerticalLocEV_ + State_ x(Test_::resol(), Test_::ctlvars(), Test_::time()); + + VerticalLocEV_ vertloc(vertlocconf, x); oops::Log::test() << "Number of eigenvalues used in VerticalLoc: " << vertloc.neig() << std::endl; //--- check for expected number of eigen modes @@ -94,10 +100,12 @@ template void testVerticalLocEV() { // check the orthogonality condition double n0 = incEns[0][0].dot_product_with(incEns[0][0]); oops::Log::debug() << "dot product 0: " << n0 << std::endl; + // check that eigen vectors are not zeros EXPECT(n0 > 0); double tol = 2*n0*DBL_EPSILON; oops::Log::debug() << "tolerance :" << tol << std::endl; + // check that eig[ieig>0] are orthogonal to eig[0] for (int i = 1; i < neig; ++i) { double n = incEns[0][0].dot_product_with(incEns[i][0]); oops::Log::debug() << "dot product " << i << ": " << n << std::endl; @@ -112,6 +120,7 @@ template void testVerticalLocEV() { Eigen::MatrixXd modInc = vertloc.modulateIncrement(incEns2, geometry.begin(), 0); Eigen::MatrixXd modIncInner = modInc.transpose()*modInc; oops::Log::debug() << "modInc'*modInc" << modIncInner << std::endl; + // modIncInner should be a diagonal matrix for (int i = 1; i < neig; ++i) { EXPECT(modIncInner(0, i) < modIncInner(0, 0)*DBL_EPSILON); } diff --git a/src/test/interface/ErrorCovariance.h b/src/test/interface/ErrorCovariance.h index d524e7b60..ce4978a36 100644 --- a/src/test/interface/ErrorCovariance.h +++ b/src/test/interface/ErrorCovariance.h @@ -23,12 +23,12 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/instantiateCovarFactory.h" #include "oops/base/ModelSpaceCovarianceBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" diff --git a/src/test/interface/GeoVaLs.h b/src/test/interface/GeoVaLs.h index 7d96f42af..47876eadd 100644 --- a/src/test/interface/GeoVaLs.h +++ b/src/test/interface/GeoVaLs.h @@ -20,6 +20,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/mpi/mpi.h" diff --git a/src/test/interface/Geometry.h b/src/test/interface/Geometry.h index 9f1af202a..9f428075c 100644 --- a/src/test/interface/Geometry.h +++ b/src/test/interface/Geometry.h @@ -19,7 +19,7 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/Logger.h" diff --git a/src/test/interface/GeometryFixture.h b/src/test/interface/GeometryFixture.h index 2fbbf72d2..aa8cb0039 100644 --- a/src/test/interface/GeometryFixture.h +++ b/src/test/interface/GeometryFixture.h @@ -12,7 +12,7 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/util/parameters/IgnoreOtherParameters.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" diff --git a/src/test/interface/GeometryIterator.h b/src/test/interface/GeometryIterator.h index 4ac4c48e0..2f3e89296 100644 --- a/src/test/interface/GeometryIterator.h +++ b/src/test/interface/GeometryIterator.h @@ -20,10 +20,10 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LocalIncrement.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/Increment.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" diff --git a/src/test/interface/GetValues.h b/src/test/interface/GetValues.h index 4b776132d..acf64e73d 100644 --- a/src/test/interface/GetValues.h +++ b/src/test/interface/GetValues.h @@ -24,13 +24,13 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/interface/AnalyticInit.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -60,6 +60,7 @@ template class GetValuesFixture : private boost:: static const LocalConfig_ & testconf() {return *getInstance().testconf_;} static const Locations_ & locs() {return *getInstance().locs_;} static const Variables_ & geovalvars() {return *getInstance().geovalvars_;} + static const std::vector & geovalvarsizes() {return getInstance().geovalvarsizes_;} static void reset() { getInstance().getvalues_.reset(); getInstance().geovals_.reset(); @@ -86,6 +87,7 @@ template class GetValuesFixture : private boost:: // Variables geovalvars_.reset(new Variables_(TestEnvironment::config(), "state variables")); + geovalvarsizes_ = resol_->variableSizes(*geovalvars_); // Locations const LocalConfig_ locsConfig(TestEnvironment::config(), "locations"); @@ -96,7 +98,7 @@ template class GetValuesFixture : private boost:: timeend_.reset(new DateTime_(locsConfig.getString("window end"))); // GeoVaLs - geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_)); + geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_, geovalvarsizes_)); // GetValues LocalConfig_ getvaluesConfig; @@ -114,6 +116,7 @@ template class GetValuesFixture : private boost:: std::unique_ptr testconf_; std::unique_ptr locs_; std::unique_ptr geovalvars_; + std::vector geovalvarsizes_; }; // ================================================================================================= @@ -146,8 +149,8 @@ template void testGetValuesMultiWindow() { EXPECT(xx.norm() > 0.0); - GeoVaLs_ gv1(Test_::locs(), Test_::geovalvars()); - GeoVaLs_ gv2(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv1(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); + GeoVaLs_ gv2(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); // Compute all geovals together Test_::getvalues().fillGeoVaLs(xx, Test_::timebeg(), Test_::timeend(), gv1); @@ -209,7 +212,7 @@ template void testGetValuesInterpolation() { double interp_tol = Test_::testconf().getDouble("interpolation tolerance"); // Ceate a GeoVaLs object from locs and vars - GeoVaLs_ gval(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gval(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); EXPECT(xx.norm() > 0.0); diff --git a/src/test/interface/Increment.h b/src/test/interface/Increment.h index f61689207..903480e65 100644 --- a/src/test/interface/Increment.h +++ b/src/test/interface/Increment.h @@ -22,12 +22,13 @@ #include +#include "atlas/field.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -35,7 +36,6 @@ #include "oops/util/Logger.h" #include "test/TestEnvironment.h" - namespace test { // ============================================================================= @@ -48,6 +48,7 @@ template class IncrementFixture : private boost::noncopyable { static const oops::Variables & ctlvars() {return *getInstance().ctlvars_;} static const util::DateTime & time() {return *getInstance().time_;} static const double & tolerance() {return getInstance().tolerance_;} + static const int & skipAtlas() {return getInstance().skipAtlas_;} static const eckit::Configuration & test() {return *getInstance().test_;} static void reset() { getInstance().time_.reset(); @@ -78,6 +79,7 @@ template class IncrementFixture : private boost::noncopyable { "Warning: Increment norm tolerance greater than 1e-8 " "may not be suitable for certain solvers." << std::endl; } + skipAtlas_ = test_->getInt("skip atlas", 0); } ~IncrementFixture() {} @@ -86,6 +88,7 @@ template class IncrementFixture : private boost::noncopyable { std::unique_ptr ctlvars_; std::unique_ptr test_; double tolerance_; + int skipAtlas_; std::unique_ptr time_; std::unique_ptr skipAccumTest_; std::unique_ptr skipDiffTest_; @@ -352,6 +355,57 @@ template void testIncrementSerialize() { // ----------------------------------------------------------------------------- +template void testIncrementAtlas() { + typedef IncrementFixture Test_; + typedef oops::Increment Increment_; + + if (Test_::skipAtlas() == 0) { + // Create random increment + Increment_ dx1(Test_::resol(), Test_::ctlvars(), Test_::time()); + dx1.random(); + std::unique_ptr atlasFieldSet1(new atlas::FieldSet()); + dx1.setAtlas(atlasFieldSet1.get()); + dx1.toAtlas(atlasFieldSet1.get()); + + // Create zero increment + Increment_ dx2(Test_::resol(), Test_::ctlvars(), Test_::time()); + dx2.zero(); + std::unique_ptr atlasFieldSet2(new atlas::FieldSet()); + dx2.setAtlas(atlasFieldSet2.get()); + + // Copy ATLAS fieldset content + for ( atlas::FieldSet::const_iterator field1 = atlasFieldSet1->cbegin(); + field1 != atlasFieldSet1->cend(); ++field1 ) { + atlas::Field field2 = atlasFieldSet2->field(field1->name()); + if (field1->rank() == 1) { + auto array1 = atlas::array::make_view(*field1); + auto array2 = atlas::array::make_view(field2); + for (int i1 = 0; i1 < array1.shape(0); ++i1) { + array2(i1) = array1(i1); + } + } + if (field1->rank() == 2) { + auto array1 = atlas::array::make_view(*field1); + auto array2 = atlas::array::make_view(field2); + for (int i1 = 0; i1 < array1.shape(0); ++i1) { + for (int i2 = 0; i2 < array1.shape(1); ++i2) { + array2(i1, i2) = array1(i1, i2); + } + } + } + } + + // Copy back to increment + dx2.fromAtlas(atlasFieldSet2.get()); + + // Check increment difference + dx2 -= dx1; + EXPECT(dx2.norm() == 0.0); + } +} + +// ----------------------------------------------------------------------------- + template void testIncrementDiff() { typedef IncrementFixture Test_; typedef oops::Increment Increment_; @@ -487,6 +541,8 @@ class Increment : public oops::Test { { testIncrementSchur(); }); ts.emplace_back(CASE("interface/Increment/testIncrementSerialize") { testIncrementSerialize(); }); + ts.emplace_back(CASE("interface/Increment/testIncrementAtlas") + { testIncrementAtlas(); }); } void clear() const override {} diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index 8e7fbf60a..83f03f90e 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -24,14 +24,14 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" -#include "oops/interface/Increment.h" #include "oops/interface/LinearGetValues.h" #include "oops/interface/Locations.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -67,6 +67,8 @@ template class LinearGetValuesFixture : private b static const State_ & state() {return *getInstance().state_;} static const Variables_ & statevars() {return *getInstance().statevars_;} static const Variables_ & geovalvars() {return *getInstance().geovalvars_;} + static const std::vector & geovalvarsizes() {return getInstance().geovalvarsizes_;} + static void reset() { getInstance().lineargetvalues_.reset(); getInstance().time_.reset(); @@ -97,6 +99,7 @@ template class LinearGetValuesFixture : private b // Variables geovalvars_.reset(new Variables_(TestEnvironment::config(), "state variables")); + geovalvarsizes_ = resol_->variableSizes(*geovalvars_); // Locations const LocalConfig_ locsConfig(TestEnvironment::config(), "locations"); @@ -107,7 +110,7 @@ template class LinearGetValuesFixture : private b timeend_.reset(new DateTime_(locsConfig.getString("window end"))); // GeoVaLs - geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_)); + geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_, geovalvarsizes_)); // Nonlinear GetValues LocalConfig_ getvaluesConfig; @@ -132,7 +135,7 @@ template class LinearGetValuesFixture : private b linearGetValuesConfig)); // Set trajectory - GeoVaLs_ gvtraj(*locs_, *geovalvars_); + GeoVaLs_ gvtraj(*locs_, *geovalvars_, geovalvarsizes_); lineargetvalues_->setTrajectory(*state_, *timebeg_, *timeend_, gvtraj); } @@ -150,6 +153,7 @@ template class LinearGetValuesFixture : private b std::unique_ptr state_; std::unique_ptr statevars_; std::unique_ptr geovalvars_; + std::vector geovalvarsizes_; }; // ================================================================================================= @@ -183,7 +187,7 @@ template void testLinearGetValuesZeroPert() { Increment_ dx(Test_::resol(), Test_::statevars(), Test_::time()); dx.zero(); - GeoVaLs_ gv(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); EXPECT(dx.norm() == 0.0); @@ -214,8 +218,8 @@ template void testLinearGetValuesLinearity() { EXPECT(dx1.norm() > 0.0); EXPECT(dx2.norm() > 0.0); - GeoVaLs_ gv1(Test_::locs(), Test_::geovalvars()); - GeoVaLs_ gv2(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv1(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); + GeoVaLs_ gv2(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); // Compute geovals Test_::lineargetvalues().fillGeoVaLsTL(dx1, Test_::timebeg(), Test_::timeend(), gv1); @@ -243,7 +247,7 @@ template void testLinearGetValuesLinearApproximat // Compute nonlinear geovals State_ xx0(Test_::state()); - GeoVaLs_ gv0(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv0(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); Test_::getvalues().fillGeoVaLs(xx0, Test_::timebeg(), Test_::timeend(), gv0); // Run tangent linear @@ -254,8 +258,8 @@ template void testLinearGetValuesLinearApproximat for (unsigned int jtest = 0; jtest < ntest; ++jtest) { Increment_ dxx(dx0); State_ xx(xx0); - GeoVaLs_ gv(Test_::locs(), Test_::geovalvars()); - GeoVaLs_ dgv(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); + GeoVaLs_ dgv(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); dxx *= zz; xx += dxx; @@ -302,7 +306,7 @@ template void testLinearGetValuesAdjoint() { Increment_ dx_in(Test_::resol(), Test_::statevars(), Test_::time()); Increment_ dx_ou(Test_::resol(), Test_::statevars(), Test_::time()); - GeoVaLs_ gv_ou(Test_::locs(), Test_::geovalvars()); + GeoVaLs_ gv_ou(Test_::locs(), Test_::geovalvars(), Test_::geovalvarsizes()); // Tangent linear dx_in.random(); diff --git a/src/test/interface/LinearModel.h b/src/test/interface/LinearModel.h index 4c025bc41..eff3d5fb2 100644 --- a/src/test/interface/LinearModel.h +++ b/src/test/interface/LinearModel.h @@ -26,20 +26,20 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/instantiateCovarFactory.h" +#include "oops/base/LinearModel.h" +#include "oops/base/Model.h" #include "oops/base/ModelSpaceCovarianceBase.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" +#include "oops/base/State.h" #include "oops/base/TrajectorySaver.h" #include "oops/base/Variables.h" -#include "oops/generic/instantiateTlmFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/LinearModel.h" -#include "oops/interface/Model.h" +#include "oops/generic/instantiateLinearModelFactory.h" #include "oops/interface/ModelAuxControl.h" #include "oops/interface/ModelAuxIncrement.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -123,7 +123,7 @@ template class LinearModelFixture : private boost::noncopyable tlConf_.reset(new eckit::LocalConfiguration(TestEnvironment::config(), "linear model")); // Setup trajectory for TL and AD - oops::instantiateTlmFactory(); + oops::instantiateLinearModelFactory(); oops::PostProcessor post; oops::PostProcessorTLAD pptraj; tlm_.reset(new LinearModel_(*resol_, *tlConf_)); diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index f1a03e46b..7ecaabd9f 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -33,12 +33,17 @@ namespace test { const char *expectConstructorToThrow = "expect constructor to throw exception with message"; +const char *expectSetTrajectoryToThrow = "expect setTrajectory to throw exception with message"; +const char *expectSimulateObsToThrow = "expect simulateObs to throw exception with message"; +const char *expectSimulateObsTLToThrow = "expect simulateObsTL to throw exception with message"; +const char *expectSimulateObsADToThrow = "expect simulateObsAD to throw exception with message"; // ----------------------------------------------------------------------------- /// \brief tests constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::LinearObsOperator LinearObsOperator_; + typedef typename LinearObsOperator_::Parameters_ LinearObsOperatorParameters_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); @@ -46,10 +51,12 @@ template void testConstructor() { std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); + LinearObsOperatorParameters_ linobsopparams; + linobsopparams.validateAndDeserialize(linobsopconf); if (!Test_::config(jj).has(expectConstructorToThrow)) { std::unique_ptr linobsop( - new LinearObsOperator_(Test_::obspace()[jj], linobsopconf)); + new LinearObsOperator_(Test_::obspace()[jj], linobsopparams)); EXPECT(linobsop.get()); oops::Log::test() << "Testing LinearObsOperator: " << *linobsop << std::endl; linobsop.reset(); @@ -57,7 +64,7 @@ template void testConstructor() { } else { // The constructor is expected to throw an exception containing the specified string. const std::string expectedMessage = Test_::config(jj).getString(expectConstructorToThrow); - EXPECT_THROWS_MSG(LinearObsOperator_(Test_::obspace()[jj], linobsopconf), + EXPECT_THROWS_MSG(LinearObsOperator_(Test_::obspace()[jj], linobsopparams), expectedMessage.c_str()); } } @@ -72,7 +79,9 @@ template void testLinearity() { typedef oops::ObsAuxIncrement ObsAuxIncr_; typedef oops::ObsAuxCovariance ObsAuxCov_; typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; typedef oops::LinearObsOperator LinearObsOperator_; + typedef typename LinearObsOperator_::Parameters_ LinearObsOperatorParameters_; typedef oops::ObsVector ObsVector_; const double zero = 0.0; @@ -81,20 +90,27 @@ template void testLinearity() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); - if (conf.has(expectConstructorToThrow)) + if (conf.has(expectConstructorToThrow) || + conf.has(expectSetTrajectoryToThrow) || + conf.has(expectSimulateObsToThrow) || + conf.has(expectSimulateObsTLToThrow)) continue; // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); - ObsOperator_ hop(Test_::obspace()[jj], obsopconf); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(Test_::obspace()[jj], obsopparams); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); - LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); + LinearObsOperatorParameters_ linobsopparams; + linobsopparams.validateAndDeserialize(linobsopconf); + LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopparams); // initialize obs bias eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); @@ -151,7 +167,9 @@ template void testAdjoint() { typedef ObsTestsFixture Test_; typedef oops::GeoVaLs GeoVaLs_; typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; typedef oops::LinearObsOperator LinearObsOperator_; + typedef typename LinearObsOperator_::Parameters_ LinearObsOperatorParameters_; typedef oops::ObsAuxControl ObsAuxCtrl_; typedef oops::ObsAuxIncrement ObsAuxIncr_; typedef oops::ObsAuxCovariance ObsAuxCov_; @@ -161,20 +179,28 @@ template void testAdjoint() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); - if (conf.has(expectConstructorToThrow)) + if (conf.has(expectConstructorToThrow) || + conf.has(expectSetTrajectoryToThrow) || + conf.has(expectSimulateObsToThrow) || + conf.has(expectSimulateObsTLToThrow) || + conf.has(expectSimulateObsADToThrow)) continue; // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); - ObsOperator_ hop(Test_::obspace()[jj], obsopconf); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(Test_::obspace()[jj], obsopparams); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); - LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); + LinearObsOperatorParameters_ linobsopparams; + linobsopparams.validateAndDeserialize(linobsopconf); + LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopparams); const double tol = conf.getDouble("linear obs operator test.tolerance AD"); // initialize bias correction @@ -239,26 +265,35 @@ template void testTangentLinear() { typedef oops::ObsAuxControl ObsAuxCtrl_; typedef oops::ObsAuxIncrement ObsAuxIncr_; typedef oops::ObsAuxCovariance ObsAuxCov_; - typedef oops::LinearObsOperator LinearObsOperator_; typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; + typedef oops::LinearObsOperator LinearObsOperator_; + typedef typename LinearObsOperator_::Parameters_ LinearObsOperatorParameters_; typedef oops::ObsVector ObsVector_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); - if (conf.has(expectConstructorToThrow)) + if (conf.has(expectConstructorToThrow) || + conf.has(expectSetTrajectoryToThrow) || + conf.has(expectSimulateObsToThrow) || + conf.has(expectSimulateObsTLToThrow)) continue; // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); - ObsOperator_ hop(Test_::obspace()[jj], obsopconf); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(Test_::obspace()[jj], obsopparams); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); - LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); + LinearObsOperatorParameters_ linobsopparams; + linobsopparams.validateAndDeserialize(linobsopconf); + LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopparams); const double tol = conf.getDouble("linear obs operator test.tolerance TL"); const double alpha = conf.getDouble("linear obs operator test.coef TL", 0.1); @@ -288,6 +323,8 @@ template void testTangentLinear() { ObsVector_ y1(Test_::obspace()[jj]); ObsVector_ y2(Test_::obspace()[jj]); ObsVector_ y3(Test_::obspace()[jj]); + ObsVector_ bias(Test_::obspace()[jj]); + bias.zero(); // create obsdatavector to hold diags oops::Variables diagvars; @@ -295,7 +332,7 @@ template void testTangentLinear() { ObsDiags_ ydiag(Test_::obspace()[jj], hop.locations(), diagvars); // y1 = hop(x0, ybias0) - hop.simulateObs(x0, y1, ybias0, ydiag); + hop.simulateObs(x0, y1, ybias0, bias, ydiag); // randomize dx and ybinc GeoVaLs_ dx(gconf, Test_::obspace()[jj], hoptl.requiredVars()); @@ -315,9 +352,10 @@ template void testTangentLinear() { ybinc *= alpha; ybias = ybias0; ybias += ybinc; + bias.zero(); // y2 = hop(x0+alpha*dx, ybias0+alpha*ybinc) - hop.simulateObs(x, y2, ybias, ydiag); + hop.simulateObs(x, y2, ybias, bias, ydiag); y2 -= y1; // y3 = hoptl(alpha*dx, alpha*ybinc) hoptl.simulateObsTL(dx, y3, ybinc); @@ -333,12 +371,101 @@ template void testTangentLinear() { // ----------------------------------------------------------------------------- +template void testException() { + typedef ObsTestsFixture Test_; + typedef oops::GeoVaLs GeoVaLs_; + typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; + typedef oops::LinearObsOperator LinearObsOperator_; + typedef typename LinearObsOperator_::Parameters_ LinearObsOperatorParameters_; + typedef oops::ObsAuxControl ObsAuxCtrl_; + typedef oops::ObsAuxIncrement ObsAuxIncr_; + typedef oops::ObsAuxCovariance ObsAuxCov_; + typedef oops::ObsVector ObsVector_; + + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); + if (conf.has(expectConstructorToThrow)) + continue; + + // Set up objects prior to throwing exceptions. + eckit::LocalConfiguration obsopconf(conf, "obs operator"); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(Test_::obspace()[jj], obsopparams); + std::string confname = "obs operator"; + if (conf.has("linear obs operator")) confname = "linear obs operator"; + eckit::LocalConfiguration linobsopconf(conf, confname); + LinearObsOperatorParameters_ linobsopparams; + linobsopparams.validateAndDeserialize(linobsopconf); + LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopparams); + eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); + typename ObsAuxCtrl_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasparams); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasparams); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasparams); + eckit::LocalConfiguration gconf(conf, "geovals"); + oops::Variables hopvars = hop.requiredVars(); + hopvars += ybias.requiredVars(); + const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); + oops::Variables diagvars; + diagvars += ybias.requiredHdiagnostics(); + + if (Test_::config(jj).has(expectSetTrajectoryToThrow)) { + // The setTrajectory method is expected to throw an exception + // containing the specified string. + const std::string expectedMessage = + Test_::config(jj).getString(expectSetTrajectoryToThrow); + EXPECT_THROWS_MSG(hoptl.setTrajectory(gval, ybias), + expectedMessage.c_str()); + // Do not continue further because setTrajectory must be run + // before simulateObsTL and simulateObsAD. + continue; + } + + if (Test_::config(jj).has(expectSimulateObsTLToThrow)) { + hoptl.setTrajectory(gval, ybias); + ObsVector_ dy1(Test_::obspace()[jj]); + GeoVaLs_ dx1(gconf, Test_::obspace()[jj], hoptl.requiredVars()); + dx1.random(); + Bobsbias.randomize(ybinc); + // The simulateObsTL method is expected to throw an exception + // containing the specified string. + const std::string expectedMessage = + Test_::config(jj).getString(expectSimulateObsTLToThrow); + EXPECT_THROWS_MSG(hoptl.simulateObsTL(dx1, dy1, ybinc), + expectedMessage.c_str()); + } + + if (Test_::config(jj).has(expectSimulateObsADToThrow)) { + hoptl.setTrajectory(gval, ybias); + ObsVector_ dy2(Test_::obspace()[jj]); + GeoVaLs_ dx2(gconf, Test_::obspace()[jj], hoptl.requiredVars()); + Bobsbias.randomize(ybinc); + dy2.random(); + dx2.zero(); + ybinc.zero(); + // The simulateObsAD method is expected to throw an exception + // containing the specified string. + const std::string expectedMessage = + Test_::config(jj).getString(expectSimulateObsADToThrow); + EXPECT_THROWS_MSG(hoptl.simulateObsAD(dx2, dy2, ybinc), + expectedMessage.c_str()); + } + } +} + +// ----------------------------------------------------------------------------- + template class LinearObsOperator : public oops::Test { typedef ObsTestsFixture Test_; + public: LinearObsOperator() {} virtual ~LinearObsOperator() {} + private: std::string testid() const override {return "test::LinearObsOperator<" + OBS::name() + ">";} @@ -353,6 +480,8 @@ class LinearObsOperator : public oops::Test { { testTangentLinear(); }); ts.emplace_back(CASE("interface/LinearObsOperator/testAdjoint") { testAdjoint(); }); + ts.emplace_back(CASE("interface/LinearObsOperator/testException") + { testException(); }); } void clear() const override { diff --git a/src/test/interface/LinearVariableChange.h b/src/test/interface/LinearVariableChange.h index 313173dfb..706ff2d0e 100644 --- a/src/test/interface/LinearVariableChange.h +++ b/src/test/interface/LinearVariableChange.h @@ -23,12 +23,12 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/LinearVariableChangeBase.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" #include "oops/generic/instantiateVariableChangeFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" diff --git a/src/test/interface/Localization.h b/src/test/interface/Localization.h index 9c1c82275..19a3cfbfc 100644 --- a/src/test/interface/Localization.h +++ b/src/test/interface/Localization.h @@ -23,11 +23,11 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Increment.h" #include "oops/base/IncrementEnsemble.h" -#include "oops/base/LocalizationBase.h" +#include "oops/base/Localization.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Increment.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -38,7 +38,7 @@ namespace test { // ----------------------------------------------------------------------------- template class LocalizationFixture : private boost::noncopyable { - typedef oops::LocalizationBase Localization_; + typedef oops::Localization Localization_; typedef oops::Geometry Geometry_; typedef oops::IncrementEnsemble Ensemble_; @@ -64,7 +64,7 @@ template class LocalizationFixture : private boost::noncopyable // Setup the localization matrix const eckit::LocalConfiguration conf(TestEnvironment::config(), "localization"); - local_ = oops::LocalizationFactory::create(*resol_, *time_, conf); + local_.reset(new Localization_(*resol_, conf)); oops::Log::test() << "Testing localization: " << *local_ << std::endl; } diff --git a/src/test/interface/Model.h b/src/test/interface/Model.h index 73f59b99f..68e1ce717 100644 --- a/src/test/interface/Model.h +++ b/src/test/interface/Model.h @@ -23,11 +23,11 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/Model.h" #include "oops/base/PostProcessor.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" +#include "oops/base/State.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" diff --git a/src/test/interface/ModelAuxControl.h b/src/test/interface/ModelAuxControl.h index 4545d5d88..b6e67e2c5 100644 --- a/src/test/interface/ModelAuxControl.h +++ b/src/test/interface/ModelAuxControl.h @@ -21,7 +21,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/interface/ModelAuxControl.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" diff --git a/src/test/interface/ModelAuxCovariance.h b/src/test/interface/ModelAuxCovariance.h index 157b71909..c39d65de5 100644 --- a/src/test/interface/ModelAuxCovariance.h +++ b/src/test/interface/ModelAuxCovariance.h @@ -21,7 +21,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/interface/ModelAuxCovariance.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" diff --git a/src/test/interface/ModelAuxIncrement.h b/src/test/interface/ModelAuxIncrement.h index 082459c95..0b947219c 100644 --- a/src/test/interface/ModelAuxIncrement.h +++ b/src/test/interface/ModelAuxIncrement.h @@ -23,7 +23,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" -#include "oops/interface/Geometry.h" +#include "oops/base/Geometry.h" #include "oops/interface/ModelAuxControl.h" #include "oops/interface/ModelAuxCovariance.h" #include "oops/interface/ModelAuxIncrement.h" diff --git a/src/test/interface/ObsDataVector.h b/src/test/interface/ObsDataVector.h index 4305b879e..ead9064d8 100644 --- a/src/test/interface/ObsDataVector.h +++ b/src/test/interface/ObsDataVector.h @@ -16,9 +16,9 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "oops/util/dot_product.h" #include "test/interface/ObsTestsFixture.h" diff --git a/src/test/interface/ObsLocalization.h b/src/test/interface/ObsLocalization.h index 0d0156532..a94924189 100644 --- a/src/test/interface/ObsLocalization.h +++ b/src/test/interface/ObsLocalization.h @@ -19,11 +19,10 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" #include "oops/base/ObsLocalizationBase.h" -#include "oops/interface/Geometry.h" +#include "oops/base/ObsVector.h" #include "oops/interface/GeometryIterator.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsVector.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "test/interface/ObsTestsFixture.h" @@ -43,7 +42,6 @@ template void testObsLocalization() { typedef ObsTestsFixture Test_; typedef oops::Geometry Geometry_; typedef oops::GeometryIterator GeometryIterator_; - typedef oops::ObsDataVector ObsDataVector_; typedef oops::ObsLocalizationBase ObsLocalization_; typedef oops::ObsSpace ObsSpace_; typedef oops::ObsVector ObsVector_; @@ -74,7 +72,6 @@ template void testObsLocalization() { ObsVector_ locvector(obspace); ObsVector_ obsvector(obspace); - ObsDataVector_ outside(obspace, obspace.obsvariables()); size_t total_tested = 0; std::vector nobs_local(nobs_local_ref.size(), 0); @@ -95,12 +92,10 @@ template void testObsLocalization() { total_tested++; size_t index = it - reference_points.begin(); locvector.ones(); - obsloc->computeLocalization(ii, outside, locvector); + obsloc->computeLocalization(ii, locvector); oops::Log::test() << "Obs localization with geometry iterator: " << ii << ": " << *ii << std::endl; - oops::Log::test() << "Mask for obs outside of localization: " << outside << std::endl; oops::Log::test() << "Localization values: " << locvector << std::endl; - locvector.mask(outside); oops::Log::test() << "Local vector nobs and reference: " << locvector.nobs() << ", " << nobs_local_ref[index] << std::endl; oops::Log::debug() << "Local vector stats lat,lon,nobs,rms: " << *ii << ", " diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index 2d6d02ef4..33c062518 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -18,13 +18,13 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsAuxControl.h" #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" -#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "oops/util/Expect.h" #include "test/interface/ObsTestsFixture.h" @@ -33,18 +33,22 @@ namespace test { const char *expectConstructorToThrow = "expect constructor to throw exception with message"; +const char *expectSimulateObsToThrow = "expect simulateObs to throw exception with message"; // ----------------------------------------------------------------------------- /// \brief tests constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration obsopconf(Test_::config(jj), "obs operator"); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); if (!Test_::config(jj).has(expectConstructorToThrow)) { - std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopconf)); + std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopparams)); EXPECT(hop.get()); oops::Log::test() << "Testing ObsOperator: " << *hop << std::endl; hop.reset(); @@ -52,7 +56,7 @@ template void testConstructor() { } else { // The constructor is expected to throw an exception containing the specified string. const std::string expectedMessage = Test_::config(jj).getString(expectConstructorToThrow); - EXPECT_THROWS_MSG(ObsOperator_(Test_::obspace()[jj], obsopconf), + EXPECT_THROWS_MSG(ObsOperator_(Test_::obspace()[jj], obsopparams), expectedMessage.c_str()); } } @@ -66,6 +70,7 @@ template void testSimulateObs() { typedef oops::ObsDiagnostics ObsDiags_; typedef oops::ObsAuxControl ObsAuxCtrl_; typedef oops::ObsOperator ObsOperator_; + typedef typename ObsOperator_::Parameters_ ObsOperatorParameters_; typedef oops::ObsVector ObsVector_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { @@ -76,7 +81,9 @@ template void testSimulateObs() { // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); - ObsOperator_ hop(Test_::obspace()[jj], obsopconf); + ObsOperatorParameters_ obsopparams; + obsopparams.validateAndDeserialize(obsopconf); + ObsOperator_ hop(Test_::obspace()[jj], obsopparams); // initialize bias correction eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); @@ -93,13 +100,27 @@ template void testSimulateObs() { // create obsvector to hold H(x) ObsVector_ hofx(Test_::obspace()[jj]); + // create obsvector to hold bias + ObsVector_ bias(Test_::obspace()[jj]); + bias.zero(); + // create diagnostics to hold HofX diags oops::Variables diagvars; diagvars += ybias.requiredHdiagnostics(); ObsDiags_ diags(Test_::obspace()[jj], hop.locations(), diagvars); // call H(x), save result in the output file as @hofx - hop.simulateObs(gval, hofx, ybias, diags); + if (Test_::config(jj).has(expectSimulateObsToThrow)) { + // The simulateObs method is expected to throw an exception + // containing the specified string. + const std::string expectedMessage = + Test_::config(jj).getString(expectSimulateObsToThrow); + EXPECT_THROWS_MSG(hop.simulateObs(gval, hofx, ybias, bias, diags), + expectedMessage.c_str()); + continue; + } else { + hop.simulateObs(gval, hofx, ybias, bias, diags); + } hofx.save("hofx"); const double tol = conf.getDouble("tolerance"); diff --git a/src/test/interface/ObsSpace.h b/src/test/interface/ObsSpace.h index 28cd2cf08..b9348802c 100644 --- a/src/test/interface/ObsSpace.h +++ b/src/test/interface/ObsSpace.h @@ -21,8 +21,8 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/ObsVector.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" @@ -47,17 +47,20 @@ template void testConstructor() { /// \brief tests that ObsSpaces created on subwindows have the same number obs as /// ObsSpaces created on the whole window template void testSubwindows() { - typedef ObsTestsFixture Test_; - typedef oops::ObsSpace ObsSpace_; - typedef oops::ObsVector ObsVector_; + typedef ObsTestsFixture Test_; + typedef oops::ObsSpace ObsSpace_; + typedef typename ObsSpace_::Parameters_ ObsSpaceParameters_; + typedef oops::ObsVector ObsVector_; util::DateTime tmid = Test_::tbgn() + (Test_::tend()-Test_::tbgn())/2; oops::Log::test() << "Testing subwindows: " << Test_::tbgn() << " to " << tmid << " and " << tmid << " to " << Test_::tend() << std::endl; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration obsconfig(Test_::config(jj), "obs space"); - ObsSpace_ obspace1(obsconfig, oops::mpi::world(), Test_::tbgn(), tmid); - ObsSpace_ obspace2(obsconfig, oops::mpi::world(), tmid, Test_::tend()); + ObsSpaceParameters_ obsparams; + obsparams.validateAndDeserialize(obsconfig); + ObsSpace_ obspace1(obsparams, oops::mpi::world(), Test_::tbgn(), tmid); + ObsSpace_ obspace2(obsparams, oops::mpi::world(), tmid, Test_::tend()); /// Create ObsVectors for each of the ObsSpaces, to compare nobs ObsVector_ ovec(Test_::obspace()[jj]); diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index 50a552d67..8c0a79c0a 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -13,14 +13,15 @@ #include #include +#include #include #define ECKIT_TESTING_SELF_REGISTER_CASES 0 #include "eckit/testing/Test.h" +#include "oops/base/ObsVector.h" #include "oops/base/Variables.h" #include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "oops/util/dot_product.h" #include "test/interface/ObsTestsFixture.h" @@ -70,6 +71,29 @@ template void testCopyConstructor() { } } +// ----------------------------------------------------------------------------- + +/// Test the constructor taking a std::unique_ptr. +template void testWrappingConstructor() { + typedef ObsTestsFixture Test_; + typedef oops::ObsVector ObsVector_; + + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + ObsVector_ ov(Test_::obspace()[jj]); + + ov.random(); + oops::Log::test() << "Printing random ObsVector: " << ov << std::endl; + + ObsVector_ other(std::make_unique(ov->obsvector()), + Test_::obspace()[jj]); + + const double ov2 = dot_product(ov, ov); + const double other2 = dot_product(other, other); + + EXPECT(ov2 == other2); + } +} + // ----------------------------------------------------------------------------- /// Test that: /// - dot_product of a random vector with self is non-zero @@ -289,11 +313,16 @@ template void testMask() { EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); /// test packEigen + // create maskvec ObsVector - mask with missing values + ObsVector_ maskvec = test; + maskvec.ones(); + maskvec.mask(mask); + // randomize the vector, and call packEigen with maskvec test.random(); - Eigen::VectorXd with_mask_vec = test.packEigen(mask); + Eigen::VectorXd with_mask_vec = test.packEigen(maskvec); // check that the size of returned Eigen Vector is consistent with size // returned by packEigenSize() - EXPECT_EQUAL(with_mask_vec.size(), test.packEigenSize(mask)); + EXPECT_EQUAL(with_mask_vec.size(), test.packEigenSize(maskvec)); oops::Log::debug() << "Local number of masked observations is: " << with_mask_vec.size() << std::endl; // check that the size is consistent with reference for this MPI task diff --git a/src/test/interface/State.h b/src/test/interface/State.h index a7373f95d..afcfdc3d1 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -23,9 +23,9 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" @@ -85,6 +85,7 @@ template void testStateConstructors() { EXPECT(xx1.get()); oops::Log::test() << "Printing State from yaml: " << *xx1 << std::endl; const double norm1 = xx1->norm(); + EXPECT(norm1 != 0); EXPECT(oops::is_close(norm1, norm, tol)); EXPECT(xx1->validTime() == vt); diff --git a/src/test/interface/VariableChange.h b/src/test/interface/VariableChange.h index bbc93e131..ff88fcd97 100644 --- a/src/test/interface/VariableChange.h +++ b/src/test/interface/VariableChange.h @@ -20,9 +20,9 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" +#include "oops/base/Geometry.h" +#include "oops/base/State.h" #include "oops/generic/instantiateVariableChangeFactory.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/State.h" #include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" diff --git a/src/test/mpi/mpi.h b/src/test/mpi/mpi.h index f7785266c..247069940 100644 --- a/src/test/mpi/mpi.h +++ b/src/test/mpi/mpi.h @@ -237,7 +237,6 @@ CASE("mpi/mpi/allGatherEigen") { CASE("mpi/mpi/exclusiveScan") { const eckit::mpi::Comm &comm = oops::mpi::world(); const size_t rank = comm.rank(); - const size_t size = comm.size(); size_t expectedResult = 0; for (size_t lowerRank = 0; lowerRank < rank; ++lowerRank) diff --git a/src/test/testinput/variables.yaml b/src/test/testinput/variables.yaml index 356e54739..3fa0def38 100644 --- a/src/test/testinput/variables.yaml +++ b/src/test/testinput/variables.yaml @@ -1 +1,2 @@ test variables: ['air_temperature', 'eastward_wind'] +empty variables: [] diff --git a/src/test/util/Parameters.h b/src/test/util/Parameters.h index cffa6bf4f..d4897cfec 100644 --- a/src/test/util/Parameters.h +++ b/src/test/util/Parameters.h @@ -40,6 +40,7 @@ #include "oops/util/parameters/ParameterTraitsAnyOf.h" #include "oops/util/parameters/ParameterTraitsScalarOrMap.h" #include "oops/util/parameters/PolymorphicParameter.h" +#include "oops/util/parameters/PropertyJsonSchema.h" #include "oops/util/parameters/RequiredParameter.h" #include "oops/util/parameters/RequiredPolymorphicParameter.h" @@ -218,6 +219,23 @@ class OptionalDeviceParameters : public oops::Parameters { device{"type", this}; }; +// Class required by tests checking JSON schema description + +class DescriptiveParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(DescriptiveParameters, Parameters) + public: + oops::Parameter target{"target", "Normal target to greet", "world", this}; + oops::RequiredParameter vipTarget{"vip_target", "VIP target to greet", this}; + oops::OptionalParameter optTarget{"opt_target", "Optional target to greet", this}; + oops::OptionalParameter voptTarget{"vopt_target", "Optional target to void", this}; + oops::PolymorphicParameter + deviceOne{"type", "Device 001", "screen", this}; + oops::RequiredPolymorphicParameter + deviceTwo{"type", "Device 002", this}; + oops::OptionalPolymorphicParameter + deviceThree{"type", "Device 003", this}; +}; + class AllDeviceParameters : public oops::Parameters { OOPS_CONCRETE_PARAMETERS(AllDeviceParameters, Parameters) public: @@ -872,6 +890,44 @@ void testSetIntParameters() { } } +// Parameters with descriptions + +void testDescriptiveParameters() { + const DescriptiveParameters params; + + const oops::ObjectJsonSchema ojsTarget = params.target.jsonSchema(); + const std::string targetPropStr = oops::toString(ojsTarget.properties().at("target")); + EXPECT(targetPropStr.find("Normal target to greet") != std::string::npos); + + const oops::ObjectJsonSchema ojsVipTarget = params.vipTarget.jsonSchema(); + const std::string vipTargetPropStr = oops::toString(ojsVipTarget.properties().at("vip_target")); + EXPECT(vipTargetPropStr.find("VIP target to greet") != std::string::npos); + + const oops::ObjectJsonSchema ojsOptTarget = params.optTarget.jsonSchema(); + const std::string optTargetPropStr = oops::toString(ojsOptTarget.properties().at("opt_target")); + EXPECT(optTargetPropStr.find("Optional target to greet") != std::string::npos); + + const oops::ObjectJsonSchema ojsVOptTarget = params.voptTarget.jsonSchema(); + const std::string voptTargetPropStr = oops::toString( + ojsVOptTarget.properties().at("vopt_target")); + EXPECT(voptTargetPropStr.find("Optional target to void") != std::string::npos); + + const oops::ObjectJsonSchema ojsDeviceOne = params.deviceOne.jsonSchema(); + const std::string deviceOnePropStr = oops::toString( + ojsDeviceOne.properties().at("type")); + EXPECT(deviceOnePropStr.find("Device 001") != std::string::npos); + + const oops::ObjectJsonSchema ojsDeviceTwo = params.deviceTwo.jsonSchema(); + const std::string deviceTwoPropStr = oops::toString( + ojsDeviceTwo.properties().at("type")); + EXPECT(deviceTwoPropStr.find("Device 002") != std::string::npos); + + const oops::ObjectJsonSchema ojsDeviceThree = params.deviceThree.jsonSchema(); + const std::string deviceThreePropStr = oops::toString( + ojsDeviceThree.properties().at("type")); + EXPECT(deviceThreePropStr.find("Device 003") != std::string::npos); +} + // Parameters storing Variables objects void testVariablesDeserializationWithoutChannels() { @@ -1541,6 +1597,17 @@ void testValidateAndDeserialize() { if (validationSupported) EXPECT_THROWS(oops::validateAndDeserialize(conf)); } + + + { + std::vector confs = + TestEnvironment::config().getSubConfigurations("full.range_parameters"); + std::vector ranges = oops::validateAndDeserialize(confs); + EXPECT_EQUAL(ranges[0].minParameter, 9); + EXPECT_EQUAL(ranges[0].maxParameter, 10); + EXPECT_EQUAL(ranges[1].minParameter, 11); + EXPECT_EQUAL(ranges[1].maxParameter, 12); + } } // HasParameters_ @@ -1647,6 +1714,10 @@ class Parameters : public oops::Test { testSetIntParameters(); }); + ts.emplace_back(CASE("util/Parameters/testDescriptiveParameters") { + testDescriptiveParameters(); + }); + ts.emplace_back(CASE("util/Parameters/testVariablesDeserializationWithoutChannels") { testVariablesDeserializationWithoutChannels(); }); diff --git a/src/test/util/TestReference.h b/src/test/util/TestReference.h index 7bda5de85..b2c5d2b84 100644 --- a/src/test/util/TestReference.h +++ b/src/test/util/TestReference.h @@ -18,11 +18,12 @@ namespace test { CASE("util/TestReference") { std::string test1 = "Line 1: +0123456789\n"; std::string test2 = "Line 1: +0123456789\nLine 2: +99.98764321e-3\n"; + std::string test2_1 = "\n\nLine 1: +0123456789\n\n\nLine 2: +99.98764321e-3\n\n\n\n"; std::string test3 = "-ABC-XYZ-\n"; std::string test4 = "-99-+EE-++44E0\n"; // Parsed as {"-99", "+44E0"} std::string good_ref1 = "Line 1: 123456789\n"; - std::string good_ref2 = "Line 1: 123456789\nLine 2: 9.9987643212e-2\n"; + std::string good_ref2 = "Line 1: 123456789\nLine 2: 9.9987643212e-2\n\n"; std::string good_ref4 = "-9.90000E1-+EE-+44\n"; std::string bad_ref1 = "Line 1: -123456789\n"; @@ -40,18 +41,23 @@ CASE("util/TestReference") { EXPECT_NO_THROW((oops::TestReference::compare(test1, good_ref1, fTol, iTol))); EXPECT_NO_THROW((oops::TestReference::compare(test2, good_ref2, fTol, iTol))); + EXPECT_NO_THROW((oops::TestReference::compare(test2_1, good_ref2, fTol, iTol))); EXPECT_THROWS_AS((oops::TestReference::compare(test1, good_ref2, fTol, iTol)), oops::TestReferenceMissingTestLineError); EXPECT_THROWS_AS((oops::TestReference::compare(test2, good_ref1, fTol, iTol)), oops::TestReferenceMissingReferenceLineError); + EXPECT_THROWS_AS((oops::TestReference::compare(test2_1, good_ref1, fTol, iTol)), + oops::TestReferenceMissingReferenceLineError); EXPECT_THROWS_AS((oops::TestReference::compare(test1, bad_ref1, fTol, iTol)), oops::TestReferenceIntegerMismatchError); EXPECT_THROWS_AS((oops::TestReference::compare(test2, bad_ref2_1, fTol, iTol)), oops::TestReferenceFloatMismatchError); + EXPECT_THROWS_AS((oops::TestReference::compare(test2_1, bad_ref2_1, fTol, iTol)), + oops::TestReferenceFloatMismatchError); EXPECT_THROWS_AS((oops::TestReference::compare(test2, bad_ref2_2, fTol, iTol)), oops::TestReferenceTextMismatchError); diff --git a/src/test/util/algorithms.cc b/src/test/util/algorithms.cc new file mode 100644 index 000000000..a58e23c46 --- /dev/null +++ b/src/test/util/algorithms.cc @@ -0,0 +1,15 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/runs/Run.h" +#include "test/util/algorithms.h" + +int main(int argc, char **argv) { + oops::Run run(argc, argv); + test::Algorithms tests; + return run.execute(tests); +} diff --git a/src/test/util/algorithms.h b/src/test/util/algorithms.h new file mode 100644 index 000000000..415b90c86 --- /dev/null +++ b/src/test/util/algorithms.h @@ -0,0 +1,45 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UTIL_ALGORITHMS_H_ +#define TEST_UTIL_ALGORITHMS_H_ + +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/../test/TestEnvironment.h" +#include "oops/runs/Test.h" +#include "oops/util/algorithms.h" +#include "oops/util/Expect.h" + +namespace test { + +CASE("util/Algorithms/transformVector") { + const std::vector input1{1, 2, 3}; + const std::vector output1 = + util::transformVector(input1, [](int x) { return std::to_string(x); }); + const std::vector expectedOutput1{"1", "2", "3"}; + EXPECT_EQUAL(output1, expectedOutput1); + + const std::vector input2; + const std::vector output2 = + util::transformVector(input2, [](int x) { return std::to_string(x); }); + EXPECT(output2.empty()); +} + +class Algorithms : public oops::Test { + private: + std::string testid() const override { return "test::algorithms"; } + void register_tests() const override {} + void clear() const override {} +}; + +} // namespace test + +#endif // TEST_UTIL_ALGORITHMS_H_ diff --git a/src/test/util/signal_trap.cc b/src/test/util/signal_trap.cc new file mode 100644 index 000000000..5e6d5e934 --- /dev/null +++ b/src/test/util/signal_trap.cc @@ -0,0 +1,27 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +/* + * This test deliberately produces a floating point exception. Not using eckit + * because our fpe handlers always produce an abort() call. ctest will expect + * this test to always fail. + */ +#include + +extern void trap_sigfpe(int); + +int main(int argc, char **argv) { + trap_sigfpe(1); + std::cout << "Test to trigger an fpe division by zero." << std::endl; + + for (int i=10; i >= 0; i--) { + std::cout << "10 / " << i << " = "; + double res = 10. / static_cast(i); + std::cout << res << std::endl; + } + return 0; +} + diff --git a/tools/compare.py b/tools/compare.py index 7864824b1..469f3dc06 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -64,7 +64,7 @@ def line_diff(line1,line2,lnum,ftol,idif): found=found+1 flt1a = reflte.findall(flt1[0]) flt2a = reflte.findall(flt2[0]) - rdiff = abs(float(flt1a[0])-float(flt2a[0]))/(float(flt1a[0])+1.0e-6) + rdiff = abs(float(flt1a[0])-float(flt2a[0]))/(abs(float(flt1a[0]))+1.0e-6) if (not rdiff <= ftol): lineerror=lineerror+1 print("Float mismatch at line "+str(lnum)+": "+\ @@ -97,7 +97,7 @@ def line_diff(line1,line2,lnum,ftol,idif): found=found+1 flt1a = reflte.findall(flt1[0]) int2a = reinte.findall(int2[0]) - rdiff = abs(float(flt1a[0])-float(int2a[0]))/(float(flt1a[0])+1.0e-6) + rdiff = abs(float(flt1a[0])-float(int2a[0]))/(abs(float(flt1a[0]))+1.0e-6) if (not rdiff <= ftol): lineerror=lineerror+1 print("Float mismatch at line "+str(lnum)+": "+\ @@ -108,14 +108,14 @@ def line_diff(line1,line2,lnum,ftol,idif): found=found+1 int1a = reinte.findall(int1[0]) flt2a = reflte.findall(flt2[0]) - rdiff = abs(float(int1a[0])-float(flt2a[0]))/(float(int1a[0])+1.0e-6) + rdiff = abs(float(int1a[0])-float(flt2a[0]))/(abs(float(int1a[0]))+1.0e-6) if (not rdiff <= ftol): lineerror=lineerror+1 print("Float mismatch at line "+str(lnum)+": "+\ int1a[0]," not equal to ",flt2a[0]," with max relative difference ", ftol,\ " Actual relative difference = ",rdiff) - #Exit with error if check has failed + #Exit with error if check has failed if found == 0: if sline1[n] != sline2[n]: print("In looping through line elements did not match either non numeric, float, date or integer. Error at line "\