Merge pull request #261 from wtsi-npg/devel

Pull from devel to master to create release 3.3.0

.github/workflows/unit-tests.yml

@@ -25,17 +25,38 @@ jobs:
           - irods: "4.2.7"
             server_image: "wsinpg/ub-18.04-irods-4.2.10:latest"
             experimental: true
+          - irods: "4.2.7"
+            server_image: "wsinpg/ub-18.04-irods-4.2.11:latest"
+            experimental: true
 
           - irods: "4.2.10"
             server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
             experimental: true
           - irods: "4.2.10"
             server_image: "wsinpg/ub-18.04-irods-4.2.10:latest"
             experimental: false
+          - irods: "4.2.10"
+            server_image: "wsinpg/ub-18.04-irods-4.2.11:latest"
+            experimental: true
+
+          - irods: "4.2.11"
+            server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
+            experimental: true
+          - irods: "4.2.11"
+            server_image: "wsinpg/ub-18.04-irods-4.2.10:latest"
+            experimental: true
+          - irods: "4.2.11"
+            server_image: "wsinpg/ub-18.04-irods-4.2.11:latest"
+            experimental: false
+
 
     services:
       irods:
         image: ${{ matrix.server_image }}
+        options: --name irods-server # A consistent container name is necessary
+                                     # to allow creation of bad replicas.
+                                     # 'irods-server' is the default container
+                                     # name in the bad replica script
         ports:
           - 1247:1247
           - 20000-20199:20000-20199

ChangeLog

@@ -1,5 +1,18 @@
 	[Upcoming]
 
+	[3.3.0]
+
+	Fix ability to list the checksum of an object with a bad replica.
+
+	Add iRODS 4.2.11 to test matrix.
+
+	Improve signal handling so that iRODS connections are closed
+	before exiting.
+
+	Fix a segfault caused by irods error values being returned as if
+	they were the number of bytes read when a user did not have
+	permission to get an object.
+
 	[3.2.0]
 
 	Fix a segfault when the file specified by the -f/--file option was

src/Makefile.am

@@ -27,6 +27,7 @@ libbaton_include_HEADERS = baton.h \
                            operations.h \
                            query.h \
                            read.h \
+                           signal_handler.h \
                            utilities.h \
                            write.h
 
@@ -40,6 +41,7 @@ libbaton_la_SOURCES = baton.c \
                       operations.c \
                       query.c \
                       read.c \
+                      signal_handler.c \
                       utilities.c \
                       write.c
 

src/baton.c

@@ -32,6 +32,7 @@
 
 #include "config.h"
 #include "baton.h"
+#include "signal_handler.h"
 
 static const char *metadata_op_name(metadata_op op) {
     const char *name;
@@ -69,23 +70,14 @@ static rcComm_t *rods_connect(rodsEnv *env){
     rcComm_t *conn = NULL;
     rErrMsg_t errmsg;
 
-    // Override the signal handler installed by the iRODS client
-    // library (all versions up to 4.1.7, inclusive) because it
-    // detects the signal and leaves the program running in a tight
-    // loop. Here we ignore SIGPIPE and fail on read/write.
-    struct sigaction saction;
-    saction.sa_handler = SIG_IGN;
-    saction.sa_flags   = 0;
-    sigemptyset(&saction.sa_mask);
-
     // TODO: add option for NO_RECONN vs. RECONN_TIMEOUT
     conn = rcConnect(env->rodsHost, env->rodsPort, env->rodsUserName,
                      env->rodsZone, NO_RECONN, &errmsg);
+
     if (!conn) goto finally;
 
-    int sigstatus = sigaction(SIGPIPE, &saction, NULL);
+    int sigstatus = apply_signal_handler();
     if (sigstatus != 0) {
-        logmsg(FATAL, "Failed to set the iRODS client SIGPIPE handler");
         exit(1);
     }
 

src/list.c

@@ -212,6 +212,7 @@ json_t *list_checksum(rcComm_t *conn, rodsPath_t *rods_path,
     query_in = make_query_input(SEARCH_MAX_ROWS, obj_format.num_columns,
                                 obj_format.columns);
     query_in = prepare_obj_list(query_in, rods_path, NULL);
+    query_in = limit_to_good_repl(query_in);
 
     results = do_query(conn, query_in, obj_format.labels, error);
     if (error->code != 0) goto error;

src/operations.c

@@ -34,7 +34,7 @@ static int iterate_json(FILE *input, rodsEnv *env, baton_json_op fn,
     int drop_conn_count = 0;
     int        status   = 0;
 
-    while (!feof(input)) {
+    while (!exit_flag && !feof(input)) {
         size_t jflags = JSON_DISABLE_EOF_CHECK | JSON_REJECT_DUPLICATES;
         json_error_t load_error;
         json_t *item = json_loadf(input, jflags, &load_error); // JSON alloc
@@ -154,6 +154,12 @@ static int iterate_json(FILE *input, rodsEnv *env, baton_json_op fn,
         }
     } // while
 
+    if (exit_flag) {
+      status = exit_flag;
+      logmsg(WARN, "Exiting on signal with code %d", exit_flag);
+      goto finally;
+    }
+
     if (drop_conn_count > 0) {
       logmsg(WARN, "Reconnected for put operations %d times",
              drop_conn_count);
@@ -168,30 +174,35 @@ static int iterate_json(FILE *input, rodsEnv *env, baton_json_op fn,
 int do_operation(FILE *input, baton_json_op fn, operation_args_t *args) {
     int item_count  = 0;
     int error_count = 0;
-
+    int status      = 0;
+
     rodsEnv env;
 
-    if (!input) goto error;
+    if (!input) {
+      status = 1;
+      goto error;
+    }
 
-    int status = iterate_json(input, &env, fn, args, &item_count, &error_count);
+    status = iterate_json(input, &env, fn, args, &item_count, &error_count);
     if (status != 0) goto error;
 
     if (error_count > 0) {
         logmsg(WARN, "Processed %d items with %d errors",
                item_count, error_count);
+	status = 1;
     }
     else {
         logmsg(DEBUG, "Processed %d items with %d errors",
                item_count, error_count);
     }
 
-    return error_count;
+    return status;
 
 error:
     logmsg(ERROR, "Processed %d items with %d errors",
            item_count, error_count);
 
-    return 1;
+    return status;
 }
 
 json_t *baton_json_dispatch_op(rodsEnv *env, rcComm_t *conn, json_t *envelope,

src/operations.h

@@ -27,6 +27,7 @@
 #include <jansson.h>
 
 #include "config.h"
+#include "signal_handler.h"
 
 /**
  *  @enum metadata_op
@@ -131,7 +132,8 @@ typedef json_t *(*baton_json_op) (rodsEnv *env,
  * @param[in]  zone_name    Zone name, char * (optional).
  * @param[in]  buffer_size  Data transfer buffer, size_t (optional).
  *
- * @return 0 on success, iRODS error code on failure.
+ * @return 0 on success, error code on failure. The error code is suitable
+ * for use as an exit code for the program calling do_operation.
  */
 int do_operation(FILE *input, baton_json_op fn, operation_args_t *args);
 

src/read.c

@@ -233,6 +233,7 @@ size_t read_chunk(rcComm_t *conn, data_obj_file_t *data_obj, char *buffer,
         set_baton_error(error, num_read,
                         "Failed to read up to %zu bytes from '%s': %s",
                         len, data_obj->path, err_name);
+        num_read = 0;
         goto finally;
     }
 

src/signal_handler.c

@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2021 Genome
+ * Research Ltd. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @file signal_handler.c
+ * @author Michael Kubiak <[email protected]>
+ */
+
+#include "signal_handler.h"
+#include "baton.h"
+
+#include <signal.h>
+
+int signals[] = {SIGINT, SIGQUIT, SIGHUP, SIGTERM, SIGUSR1, SIGUSR2, SIGPIPE, 0};
+int exit_flag;
+
+void handle_signal(int signal){
+  switch (signal) {
+  case SIGINT:
+    exit_flag = 2;
+    break;
+  case SIGQUIT:
+    exit_flag = 3;
+    break;
+  case SIGHUP:
+    exit_flag = 4;
+    break;
+  case SIGTERM:
+  case SIGUSR1:
+  case SIGUSR2:
+    exit_flag = 5;
+    break;
+  case SIGPIPE:
+    exit_flag = 6;
+    break;
+  default:
+    exit_flag = 1;
+  }
+}
+
+int apply_signal_handler() {
+    exit_flag = 0;
+
+    struct sigaction saction;
+    saction.sa_handler = &handle_signal;
+    saction.sa_flags = 0;
+    sigemptyset(&saction.sa_mask);
+    int sigstatus;
+
+    // Exit gracefully on fatal signals
+    for (int i = 0; signals[i] != 0; i++) {
+        sigstatus = sigaction(signals[i], &saction, NULL);
+        if (sigstatus != 0) {
+            logmsg(FATAL, "Failed to set the iRODS client handler for signal %s", strsignal(signals[i]));
+            return -1;
+        }
+    }
+
+    return 0;
+}

src/signal_handler.h

@@ -0,0 +1,24 @@
+/**
+ * Copyright (C) 2021 Genome
+ * Research Ltd. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @file signal_handler.h
+ * @author Michael Kubiak <[email protected]>
+ */
+
+extern int exit_flag;
+void handle_signal(int signal);
+int apply_signal_handler();