Prepare ecFlow 5.12.2

.github/ci-config.yml

@@ -1,4 +1,5 @@
-cmake_options: -DBOOST_ROOT=${BOOST_ROOT_DIR} -DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIR} -DBOOST_LIBRARYDIR=${BOOST_LIB_DIR} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DENABLE_STATIC_BOOST_LIBS=OFF
+cmake_options: -DENABLE_ALL_TESTS=ON -DBOOST_ROOT=${BOOST_ROOT_DIR} -DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIR} -DBOOST_LIBRARYDIR=${BOOST_LIB_DIR} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DENABLE_STATIC_BOOST_LIBS=OFF
+ctest_options: -L nightly -E s_test|s_zombies
 dependencies: |
   ecmwf/ecbuild
 dependency_branch: develop

.github/ci-hpc-config.yml

@@ -7,3 +7,7 @@ build:
   dependencies:
     - ecmwf/ecbuild@develop
   parallel: 64
+  cmake_options:
+    - -DENABLE_ALL_TESTS=ON
+  ctest_options:
+    - -L nightly

ANode/src/ecflow/node/TaskScriptGenerator.cpp

@@ -234,15 +234,17 @@ std::string TaskScriptGenerator::getDefaultTemplateEcfFile() const {
 
 void TaskScriptGenerator::generate_head_file() const {
     std::string path = ecf_include_ + "/head.h";
-    if (fs::exists(path))
+    if (fs::exists(path)) {
+        std::cout << "Skipping generation of head file: " << path << " as it already exists\n";
         return;
+    }
 
     std::string client_exe = "%ECF_CLIENT_EXE_PATH:";
     client_exe += Ecf::CLIENT_NAME();
     client_exe += "% ";
 
     std::string contents;
-    contents += "#!/bin/ksh\n";
+    contents += "#!/usr/bin/env bash\n";
     contents += "set -e          # stop the shell on first error X\n";
     contents += "set -u          # fail when using an undefined variable\n";
     contents += "set -o pipefail # fail if last(rightmost) command exits with a non-zero status\n";
@@ -266,7 +268,7 @@ void TaskScriptGenerator::generate_head_file() const {
         "# SANITY Check, typically only valid for new platforms. make sure hostname is resolvable to an IP address\n";
     contents += "os_name=$(uname -s)\n";
     contents += "if [[ $os_name = Linux ]] ; then\n";
-    contents += "   host %ECF_HOST%\n";
+    contents += "   ping -c 1 %ECF_HOST%\n";
     contents += "fi\n";
     contents += "\n";
     contents += "# Tell ecFlow we have started\n";
@@ -295,12 +297,16 @@ void TaskScriptGenerator::generate_head_file() const {
         ss << "TaskScriptGenerator::generate_tail_file: Could not create head.h " << path << " " << errorMsg;
         throw std::runtime_error(ss.str());
     }
+
+    std::cout << "Generated header file: " << path << "\n";
 }
 
 void TaskScriptGenerator::generate_tail_file() const {
     std::string path = ecf_include_ + "/tail.h";
-    if (fs::exists(path))
+    if (fs::exists(path)) {
+        std::cout << "Skipping generation of tail file: " << path << " as it already exists\n";
         return;
+    }
 
     std::string contents = "%ECF_CLIENT_EXE_PATH:";
     contents += Ecf::CLIENT_NAME();
@@ -315,6 +321,8 @@ void TaskScriptGenerator::generate_tail_file() const {
         ss << "TaskScriptGenerator::generate_tail_file: Could not create tail.h " << path << " " << errorMsg;
         throw std::runtime_error(ss.str());
     }
+
+    std::cout << "Generated tail file: " << path << "\n";
 }
 
 } // namespace ecf

CMakeLists.txt

@@ -25,7 +25,7 @@ endif()
 find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild ) # Before project()  
 
 # this will generate variables, see ACore/ecflow_version.h.in
-project( ecflow LANGUAGES CXX VERSION 5.12.1 )
+project( ecflow LANGUAGES CXX VERSION 5.12.2 )
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
 

Pyext/ecflow/__init__.py

@@ -15,6 +15,6 @@
 The ecFlow python module
 """
 
-__version__ = '5.12.1'
+__version__ = '5.12.2'
 
 # http://stackoverflow.com/questions/13040646/how-do-i-create-documentation-with-pydoc

Pyext/src/ecflow/python/ClientDoc.cpp

@@ -680,13 +680,18 @@ const char* ClientDoc::ping() {
 }
 
 const char* ClientDoc::stats() {
-    return "Returns the `ecflow_server`_ statistics as a string\n::\n\n"
-           "   string stats()\n"
+    return "Returns the `ecflow_server`_ statistics as a string\n\n"
+           ".. warning::\n\n"
+           "    When called without arguments, this function will print the statistics to :code:`stdout`,"
+           " before returning the information as a string.\n"
+           "    To avoid printing the output, set the boolean flag :code:`to_stdout` to :code:`False`.\n"
            "\nUsage:\n\n"
            ".. code-block:: python\n\n"
            "   try:\n"
            "       ci = Client()  # use default host(ECF_HOST) & port(ECF_PORT)\n"
-           "       stats = ci.stats()\n"
+           "       stats = ci.stats()      # prints to stdout\n"
+           "       stats = ci.stats(True)  # prints to stdout\n"
+           "       stats = ci.stats(False) # does not print to stdout\n"
            "       print(stats)\n"
            "   except RuntimeError, e:\n"
            "       print(str(e))\n";

Pyext/src/ecflow/python/ExportClient.cpp

@@ -130,13 +130,16 @@ class CliSetter {
 private:
     ClientInvoker* _self;
 };
-const std::string& stats(ClientInvoker* self) {
+
+const std::string& stats(ClientInvoker* self, bool to_stdout = true) {
     self->stats();
-    // The statistics are printed to stdout for backward compatibility with previous python client versions
-    // TODO: remove printing to stdout in the next release
-    std::cout << self->server_reply().get_string() << std::endl;
+    if (to_stdout) {
+        std::cout << self->server_reply().get_string() << std::endl;
+    }
     return self->server_reply().get_string();
 }
+BOOST_PYTHON_FUNCTION_OVERLOADS(stats_overloads, stats, 1, 2)
+
 void stats_reset(ClientInvoker* self) {
     self->stats_reset();
 }
@@ -577,7 +580,7 @@ void export_Client() {
         .def("free_all_dep", &free_all_dep, ClientDoc::free_all_dep())
         .def("free_all_dep", &free_all_dep1)
         .def("ping", &ClientInvoker::pingServer, ClientDoc::ping())
-        .def("stats", &stats, return_value_policy<copy_const_reference>(), ClientDoc::stats())
+        .def("stats", &stats, stats_overloads(args("to_stdout"), ClientDoc::stats())[return_value_policy<copy_const_reference>()])
         .def("stats_reset", &stats_reset, ClientDoc::stats_reset())
         .def("get_file",
              &get_file,

Pyext/test/py_s_TestClientApi.py

@@ -14,6 +14,7 @@
 import time
 import os
 import pwd
+import select
 from datetime import datetime
 import shutil   # used to remove directory tree
 
@@ -361,8 +362,10 @@ def test_client_run(ci):
         ci.sync_local() # get the changes, synced with local defs
         suite = ci.get_defs().find_suite("test_client_run")
         assert suite is not None, "Expected to find suite test_client_run:\n" + str(ci.get_defs())
+        print(f"Suite state: {suite.get_state()}, {count}, at {datetime.now()}")
         if suite.get_state() == State.complete:
             break;
+        assert suite.get_state() != State.aborted, "Expected suite to complete, but found it was aborted"
         time.sleep(3)
         if count > 20:
             assert False, "test_client_run aborted after " + str(count) + " loops:\n" + str(ci.get_defs())
@@ -529,20 +532,66 @@ def test_client_free_dep(ci):
         time.sleep(3)       
 
     dir_to_remove = Test.ecf_home(the_port) + "/" + "test_client_free_dep"
-    shutil.rmtree(dir_to_remove, ignore_errors=True)   
+    shutil.rmtree(dir_to_remove, ignore_errors=True)
+
+
+class CustomStdOut:
+    def __enter__(self):
+        # create a pipe to store some content temporarily
+        self.read_pipe, self.write_pipe = os.pipe()
+        # save the original ouput fd
+        self.stdout = os.dup(1)
+        # redirect stdout into the write pipe
+        fd = os.dup2(self.write_pipe, 1)
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        # restore the original output fd
+        os.dup2(self.stdout, 1)
+
+    def _more_data(self):
+        # check if there is more data in the read pipe
+        r, _, _ = select.select([self.read_pipe], [], [], 0)
+        return bool(r)
+
+    def value(self):
+        out = ''
+        while self._more_data():
+            out += os.read(self.read_pipe, 1024).decode('utf-8')
+        return out
 
 
 def test_client_stats(ci):
     print_test(ci,"test_client_stats")
-    stats = ci.stats()
-    assert "statistics" in s, "Expected statistics in the response"
-
+    with CustomStdOut() as out:
+        stats = ci.stats()
+        assert "statistics" in stats, "Expected 'statistics' in the response"
+        assert "statistics" in out.value(), "Expected 'statistics' in the captured output"
+
+
+def test_client_stats_with_stdout(ci):
+    print_test(ci,"test_client_stats_with_stdout")
+    with CustomStdOut() as out:
+        stats = ci.stats(True)
+        assert "statistics" in stats, "Expected 'statistics' in the response"
+        assert "statistics" in out.value(), "Expected 'statistics' in the captured output"
+
+
+def test_client_stats_without_stdout(ci):
+    print_test(ci,"test_client_stats_without_stdout")
+    with CustomStdOut() as out:
+        stats = ci.stats(False)
+        assert "statistics" in stats, "Expected 'statistics' in the response"
+        assert not out.value() , "No captured output expected"
+
+
 def test_client_stats_reset(ci):
     print_test(ci,"test_client_stats_reset")
     ci.stats_reset()   
-    statis = ci.stats()
-    assert "statistics" in s, "Expected statistics in the response"
-
+    stats = ci.stats()
+    assert "statistics" in stats, "Expected 'statistics' in the response"
+
+
 def test_client_debug_server_on_off(ci):
     print_test(ci,"test_client_debug_server_on_off")
     ci.debug_server_on()  # writes to standard out
@@ -2058,7 +2107,9 @@ def do_tests(ci,the_port):
     test_client_check(ci)  
     test_client_check_defstatus(ci)  
 
-    test_client_stats(ci)             
+    test_client_stats(ci)
+    test_client_stats_with_stdout(ci)
+    test_client_stats_without_stdout(ci)
     test_client_stats_reset(ci)             
     test_client_debug_server_on_off(ci)    
 

docs/conf.py

@@ -93,7 +93,7 @@
 
 
 def get_ecflow_version():
-    version = "5.12.1"
+    version = "5.12.2"
     ecflow_version = version.split(".")
     print("Extracted ecflow version '" + str(ecflow_version))
     return ecflow_version

docs/python_api/Client.rst

@@ -2259,21 +2259,25 @@ Usage:
 sort_attributes( (Client)arg1, (list)paths, (str)attribute_name [, (bool)recursive=True]) -> None
 
 
-.. py:method:: Client.stats( (Client)arg1) -> str :
+.. py:method:: Client.stats( (Client)arg1 [, (bool)to_stdout]) -> str :
    :module: ecflow
 
 Returns the :term:`ecflow_server` statistics as a string
-::
 
-   string stats()
+.. warning::
+
+    When called without arguments, this function will print the statistics to :code:`stdout`, before returning the information as a string.
+    To avoid printing the output, set the boolean flag :code:`to_stdout` to :code:`False`.
 
 Usage:
 
 .. code-block:: python
 
    try:
        ci = Client()  # use default host(ECF_HOST) & port(ECF_PORT)
-       stats = ci.stats()
+       stats = ci.stats()      # prints to stdout
+       stats = ci.stats(True)  # prints to stdout
+       stats = ci.stats(False) # does not print to stdout
        print(stats)
    except RuntimeError, e:
        print(str(e))

docs/release_notes/index.rst

@@ -4,12 +4,7 @@ Release notes
 .. toctree::
     :maxdepth: 1
 
-    version_5.12.1
     version_5.12
-    version_5.11.4
-    version_5.11.3
-    version_5.11.2
-    version_5.11.1
     version_5.11
     version_5.10
     version_5.9

docs/release_notes/version_5.10.rst

@@ -9,6 +9,7 @@ Version 5.10.0
 
 * `Released <https://confluence.ecmwf.int/display/ECFLOW/Releases>`__\  on 2023-02-09
 
+
 REST API
 --------