Merge branch 'release/0.5.1'

.github/ci-config.yml

@@ -0,0 +1,9 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/metkit
+  ecmwf/fdb
+dependency_branch: develop
+parallelism_factor: 8

.github/ci-hpc-config.yml

@@ -0,0 +1,16 @@
+build:
+  modules:
+    - ninja
+    - aec
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/metkit@develop
+    - ecmwf/fdb@develop
+  dependency_cmake_options:
+    - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
+  parallel: 64
+  env:
+    - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+    - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

.github/workflows/cd.yml

@@ -0,0 +1,11 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  deploy:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2
+    secrets: inherit

.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'master'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      gribjump: ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+  # Run CI of private downstream packages on self-hosted runners
+  private-downstream-ci:
+    name: private-downstream-ci
+    needs: [downstream-ci]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci
+          payload: '{"gribjump": "ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      gribjump: ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
+
+  # Run CI of private downstream packages on HPC
+  private-downstream-ci-hpc:
+    name: private-downstream-ci-hpc
+    needs: [downstream-ci-hpc]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci-hpc
+          payload: '{"gribjump": "ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}"}'

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

CMakeLists.txt

@@ -40,6 +40,10 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT)
     message(FATAL_ERROR "AEC version is too old. Minimum supported version is 1.1.1. Found version: ${AEC_VERSION}")
   endif()
 
+  # Optional dependency: dhskit
+  ecbuild_find_package( NAME dhskit VERSION 0.8.6 )
+  set(GRIBJUMP_HAVE_DHSKIT ${dhskit_FOUND})
+
 endif()
 
 ########################################################################################################################
@@ -66,13 +70,15 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT)
   add_subdirectory( tests )
 endif()
 
+########################################################################################################################
+# PLUGIN
 # Write and install the manifest file for the plugin
 set( gribjump_plugin_file share/plugins/gribjump-plugin.yml )
 file( WRITE  ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "plugin:\n" )
 file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  name:      gribjump-plugin\n" )
 file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  namespace: int.ecmwf\n" )
 file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  url:       http://www.ecmwf.int\n" )
-file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  version:   ${${PROJECT_NAME}_VERSION}\n" ) # TODO set version
+file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  version:   ${${PROJECT_NAME}_VERSION}\n" )
 file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "  library:   gribjump\n" )
 
 install( FILES ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} DESTINATION share/plugins )

VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

src/gribjump/CMakeLists.txt

@@ -125,6 +125,10 @@ if( HAVE_GRIBJUMP_LOCAL_EXTRACT )
 
 endif()
 
+if (GRIBJUMP_HAVE_DHSKIT)
+list( APPEND SERVER_LIBS dhskit )
+endif()
+
 ecbuild_add_library(
 
     TARGET  gribjump

src/gribjump/gribjump_config.h.in

@@ -7,5 +7,6 @@
 // features
 
 #define GRIBJUMP_HAVE_FDB @GRIBJUMP_HAVE_FDB@ 
+#cmakedefine GRIBJUMP_HAVE_DHSKIT
 
 #endif // gribjump_gribjump_config_h

src/gribjump/remote/WorkQueue.cc

@@ -33,8 +33,8 @@ WorkQueue::~WorkQueue() {
 
 }
 
-WorkQueue::WorkQueue() : queue_(eckit::Resource<size_t>("$GRIBJUMP_QUEUESIZE", 1024)) {
-    int nthreads = eckit::Resource<size_t>("$GRIBJUMP_THREADS", LibGribJump::instance().config().getInt("threads", 1));
+WorkQueue::WorkQueue() : queue_(eckit::Resource<size_t>("$GRIBJUMP_QUEUESIZE;gribjumpQueueSize", 1024)) {
+    int nthreads = eckit::Resource<size_t>("$GRIBJUMP_THREADS;gribjumpThreads", LibGribJump::instance().config().getInt("threads", 1));
     eckit::Log::info() << "Starting " << eckit::Plural(nthreads, "thread") << std::endl;
     for (int i = 0; i < nthreads; ++i) {
         workers_.emplace_back([this] {

src/tools/gribjump-server.cc

@@ -8,25 +8,38 @@
  * does it submit to any jurisdiction.
  */
 
-#include "gribjump/remote/GribJumpServer.h"
-#include "eckit/runtime/Application.h"
 
 #include <unistd.h>
 #include <fstream>
 
 #include "eckit/config/Resource.h"
 #include "eckit/net/Port.h"
+#include "eckit/runtime/Application.h"
 
-namespace gribjump {
+#include "gribjump/gribjump_config.h"
+#include "gribjump/remote/GribJumpServer.h"
 
+#ifdef GRIBJUMP_HAVE_DHSKIT
+#include "dhskit/runtime/DHSApplication.h"
+#endif
+
+namespace {
+#ifdef GRIBJUMP_HAVE_DHSKIT
+using BaseApp = dhskit::DHSApplication;
+#else
+using BaseApp = eckit::Application;
+#endif
+} // namespace
+
+
+namespace gribjump {
 
-class GribJumpServerApp : public eckit::Application, public GribJumpServer {
+class GribJumpServerApp : public BaseApp, public GribJumpServer {
 public:
     GribJumpServerApp(int argc, char** argv) : 
-        Application(argc, argv), 
-        GribJumpServer(eckit::net::Port(
-            "gribJumpServer", 
-            eckit::Resource<int>("$GRIBJUMP_SERVER_PORT", LibGribJump::instance().config().getInt("server.port", 9777)
+        BaseApp(argc, argv), 
+        GribJumpServer(eckit::net::Port( // gribjumpServerPort
+            "gribjumpServer", eckit::Resource<int>("$GRIBJUMP_SERVER_PORT", LibGribJump::instance().config().getInt("server.port", 9777)
         )))
         {}
 
@@ -48,7 +61,6 @@ class GribJumpServerApp : public eckit::Application, public GribJumpServer {
 
 
 int main(int argc, char** argv) {
-    eckit::Log::info() << "Starting gribjump server" << std::endl;
     gribjump::GribJumpServerApp app(argc, argv);
     app.start();
     return 0;

tests/CMakeLists.txt

@@ -95,7 +95,7 @@ ecbuild_add_test(
     SOURCES "test_engine.cc"
     INCLUDES "${ECKIT_INCLUDE_DIRS}"
     ENVIRONMENT "${gribjump_env}"
-    TEST_DEPENDS gribjump_test_O1280_data_files
+    TEST_DEPENDS gribjump_test_data_files
     NO_AS_NEEDED
     LIBS gribjump
 )