From 4afa6cf5559956ba51fde520724644ecba42dcf9 Mon Sep 17 00:00:00 2001 From: Keith James Date: Wed, 29 Jun 2022 10:27:16 +0100 Subject: [PATCH 1/7] Add minimal Docker setup to enable a build and test run --- Dockerfile | 62 +++++++++++++++++++++++++++++ README | 11 +++++ docker-compose.yml | 20 ++++++++++ tests/.irods/irods_environment.json | 9 +++++ 4 files changed, 102 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 tests/.irods/irods_environment.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1ac0ef5e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:bionic + +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ + apt-get update && \ + apt-get install -q -y --no-install-recommends \ + apt-utils \ + ca-certificates \ + curl \ + gpg \ + gpg-agent \ + lsb-release \ + locales && \ + locale-gen en_GB en_GB.UTF-8 && \ + localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 + +ENV LANG=en_GB.UTF-8 \ + LANGUAGE=en_GB \ + LC_ALL=en_GB.UTF-8 + +ENV IRODS_VERSION=4.2.11-1~bionic + +RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\ + tee /etc/apt/sources.list.d/renci-irods.list && \ + apt-get update && \ + apt-get install -q -y --no-install-recommends \ + irods-dev="$IRODS_VERSION" \ + irods-runtime="$IRODS_VERSION" \ + irods-icommands="$IRODS_VERSION" + +RUN apt-get update && \ + apt-get install -q -y --no-install-recommends \ + autoconf \ + automake \ + build-essential \ + check \ + gdb \ + git \ + jq \ + lcov \ + less \ + libjansson-dev \ + libtool \ + pkg-config \ + ssh \ + valgrind \ + unattended-upgrades && \ + unattended-upgrade -d -v + +ARG USER=baton +ARG UID=1000 +ARG GID=$UID + +RUN groupadd --gid $GID $USER && \ + useradd --uid $UID --gid $GID --shell /bin/bash --create-home $USER + +USER $USER + +ENV CPPFLAGS="-I/usr/include/irods" \ + CK_DEFAULT_TIMEOUT=20 + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/README b/README index 52d0b83b..6b503afd 100644 --- a/README +++ b/README @@ -96,6 +96,17 @@ Installation: make clean install +Development: + +To set up a development environment in Docker, use the provided +docker-compose.yml and corresponding Dockerfile. These create a Linux +C development and testing environment with an accompanying iRODS server. +The recommended way to use this is by remote development in the container +using VSCode. VSCode's manual explains how to do this, see: + + https://code.visualstudio.com/docs/remote/containers + + Synopsis: For full details of the JSON accepted and returned by the programs in diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6fd0ada7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +services: + irods-server: + container_name: irods-server + image: "wsinpg/ub-18.04-irods-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}" + restart: always + ports: + - "1247:1247" + - "20000-20199:20000-20199" + + baton: + image: baton + build: + context: . + dockerfile: ./Dockerfile + volumes: + - "${PWD}:/code" + environment: + IRODS_ENVIRONMENT_FILE: "/code/tests/.irods/irods_environment.json" + depends_on: + - irods-server diff --git a/tests/.irods/irods_environment.json b/tests/.irods/irods_environment.json new file mode 100644 index 00000000..ce1799d6 --- /dev/null +++ b/tests/.irods/irods_environment.json @@ -0,0 +1,9 @@ +{ + "irods_host": "irods-server", + "irods_port": 1247, + "irods_user_name": "irods", + "irods_zone_name": "testZone", + "irods_home": "/testZone/home/irods", + "irods_default_resource": "replResc", + "irods_default_hash_scheme": "MD5" +} From 12eca1ebebfa45f2db52c3e773a4494e46b04b73 Mon Sep 17 00:00:00 2001 From: Keith James <47220353+kjsanger@users.noreply.github.com> Date: Mon, 5 Sep 2022 12:13:24 +0100 Subject: [PATCH 2/7] Add OS platform for Arm Macs Co-authored-by: mksanger <73285932+mksanger@users.noreply.github.com> --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6fd0ada7..4d365706 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: baton: image: baton + platform: linux/amd64 build: context: . dockerfile: ./Dockerfile From eba2dd72b7d2952e8609611c835c7899bdc8c2b3 Mon Sep 17 00:00:00 2001 From: Keith James Date: Fri, 2 Dec 2022 11:34:23 +0000 Subject: [PATCH 3/7] Change CI runner from Ubuntu 18.04 to ubuntu-latest Remove iRODS 4.2.10 from the test matrix. Fix compiler warnings for use of strncpy. Improve reporting on test test_regression_github_issue137. --- .github/workflows/unit-tests.yml | 27 +++++---------------------- src/query.c | 8 ++++---- tests/check_baton.c | 10 +++++++++- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9f1f46d7..c76db6a0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail defaults: @@ -19,37 +19,19 @@ jobs: strategy: matrix: include: + # iRODS 4.2.7 clients vs 4.2.7 server - irods: "4.2.7" server_image: "wsinpg/ub-16.04-irods-4.2.7:latest" experimental: false + # iRODS 4.2.11 clients vs 4.2.7 server - irods: "4.2.7" - server_image: "wsinpg/ub-18.04-irods-4.2.10:latest" - experimental: true - - irods: "4.2.7" - server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" - experimental: true - - - irods: "4.2.10" - server_image: "wsinpg/ub-16.04-irods-4.2.7:latest" - experimental: true - - irods: "4.2.10" - server_image: "wsinpg/ub-18.04-irods-4.2.10:latest" - experimental: false - - irods: "4.2.10" server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" experimental: true - - - irods: "4.2.11" - server_image: "wsinpg/ub-16.04-irods-4.2.7:latest" - experimental: true - - irods: "4.2.11" - server_image: "wsinpg/ub-18.04-irods-4.2.10:latest" - experimental: true + # iRODS 4.2.11 clients vs 4.2.11 server - irods: "4.2.11" server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" experimental: false - services: irods: image: ${{ matrix.server_image }} @@ -70,6 +52,7 @@ jobs: - name: "Configure Miniconda" run: | + conda update -n base -c defaults conda conda config --prepend pkgs_dirs "$HOME/conda/pkgs" conda config --prepend envs_dirs "$HOME/conda/envs" diff --git a/src/query.c b/src/query.c index 5683254d..afb26661 100644 --- a/src/query.c +++ b/src/query.c @@ -186,8 +186,8 @@ genQueryInp_t *prepare_obj_list(genQueryInp_t *query_in, if (!path1) goto error; if (!path2) goto error; - strncpy(path1, path, len); - strncpy(path2, path, len); + snprintf(path1, len, "%s", path); + snprintf(path2, len, "%s", path); char *coll_name = dirname(path1); char *data_name = basename(path2); @@ -282,8 +282,8 @@ genQueryInp_t *prepare_obj_repl_list(genQueryInp_t *query_in, if (!path1) goto error; if (!path2) goto error; - strncpy(path1, path, len); - strncpy(path2, path, len); + snprintf(path1, len, "%s", path); + snprintf(path2, len, "%s", path); char *coll_name = dirname(path1); char *data_name = basename(path2); diff --git a/tests/check_baton.c b/tests/check_baton.c index 18665ba1..dc7329cc 100644 --- a/tests/check_baton.c +++ b/tests/check_baton.c @@ -2689,6 +2689,8 @@ START_TEST(test_regression_github_issue137) { char *operators[] = { "=", "like", "not like", ">", "<", "n>", "n<", ">=", "<=", "n>=", "n<=" }; + int num_failed = 0; + int err_code = 0; for (size_t i = 0; i < 11; i++) { json_t *avu = json_pack("{s:s, s:s, s:s}", JSON_ATTRIBUTE_KEY, "numattr1", @@ -2698,12 +2700,18 @@ START_TEST(test_regression_github_issue137) { baton_error_t error; json_t *results = search_metadata(conn, query, NULL, flags, &error); - ck_assert_msg(error.code == 0, "failed: %s", operators[i]); + if (error.code != 0) { + num_failed++; + fprintf(stderr, "operator test failed for '%s' with code %d\n", + operators[i], error.code); + } json_decref(query); json_decref(results); } + ck_assert_msg(num_failed ==0, "failed %d operator tests", num_failed); + // Test 'in' here json_t *avu = json_pack("{s:s, s:[s], s:s}", JSON_ATTRIBUTE_KEY, "numattr1", From 4fdc5e093935264318fb7e7351962cc7593225a6 Mon Sep 17 00:00:00 2001 From: Keith James Date: Wed, 7 Dec 2022 11:11:01 +0000 Subject: [PATCH 4/7] Change CI to build in a Docker container Change the GitHub CI build to be within a Docker container. Change the test suite to avoid reliance on docker exec, which may not be available inseide docker. Remove reliance on Conda. --- .github/workflows/unit-tests.yml | 108 +++++++++---------------- tests/check_baton.c | 14 +--- tests/scripts/make_bad_replica.sh | 128 ------------------------------ tests/scripts/multi_write.sh | 26 ------ tests/sql/specific_queries.sql | 2 + 5 files changed, 44 insertions(+), 234 deletions(-) delete mode 100755 tests/scripts/make_bad_replica.sh delete mode 100755 tests/scripts/multi_write.sh diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c76db6a0..722127ee 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -5,81 +5,54 @@ on: [push, pull_request] jobs: build: runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail - - defaults: - run: - shell: bash -l -e -o pipefail {0} - env: - WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic" - CONDA_TEST_ENVIRONMENT: "testenv" - CK_DEFAULT_TIMEOUT: 20 + continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail strategy: matrix: include: - # iRODS 4.2.7 clients vs 4.2.7 server - - irods: "4.2.7" - server_image: "wsinpg/ub-16.04-irods-4.2.7:latest" - experimental: false - # iRODS 4.2.11 clients vs 4.2.7 server - - irods: "4.2.7" - server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" - experimental: true - # iRODS 4.2.11 clients vs 4.2.11 server + # iRODS 4.2.11 clients on bionic - irods: "4.2.11" + build_image: "wsinpg/ub-18.04-irods-clients-dev-4.2.11:latest" server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" experimental: false + # iRODS 4.3.0 clients on bionic + - irods: "4.3.0" + build_image: "wsinpg/ub-18.04-irods-clients-dev-4.3.0:latest" + server_image: "wsinpg/ub-18.04-irods-4.3.0:latest" + experimental: true + # iRODS 4.3.0 clients on focal + - irods: "4.3.0" + build_image: "wsinpg/ub-20.04-irods-clients-dev-4.3.0:latest" + server_image: "wsinpg/ub-18.04-irods-4.3.0:latest" + experimental: true + + container: + image: ${{ matrix.build_image }} services: - irods: + irods-server: image: ${{ matrix.server_image }} - options: --name irods-server # A consistent container name is necessary - # to allow creation of bad replicas. - # 'irods-server' is the default container - # name in the bad replica script ports: - - 1247:1247 - - 20000-20199:20000-20199 + - "1247:1247" + - "20000-20199:20000-20199" + options: >- + --health-cmd "nc -z -v localhost 1247" + --health-start-period 30s + --health-interval 10s + --health-timeout 20s + --health-retries 6 steps: - - uses: actions/checkout@v2 - - - name: "Initialize Miniconda" - run: | - echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile" - - - name: "Configure Miniconda" - run: | - conda update -n base -c defaults conda - conda config --prepend pkgs_dirs "$HOME/conda/pkgs" - conda config --prepend envs_dirs "$HOME/conda/envs" - - conda config --set auto_update_conda False - conda config --prepend channels "$WSI_CONDA_CHANNEL" - conda info - - - name: "Install iRODS clients" - run: | - conda create -y -n "$CONDA_TEST_ENVIRONMENT" - conda install -y -n "$CONDA_TEST_ENVIRONMENT" python=3.9 - conda install -y -n "$CONDA_TEST_ENVIRONMENT" sphinx=2.4 - - conda install -y -n "$CONDA_TEST_ENVIRONMENT" check - conda install -y -n "$CONDA_TEST_ENVIRONMENT" libjansson-dev - conda install -y -n "$CONDA_TEST_ENVIRONMENT" "irods-dev ==${{ matrix.irods }}" - - conda install -y -n "$CONDA_TEST_ENVIRONMENT" "irods-icommands ==${{ matrix.irods }}" + - name: Checkout + uses: actions/checkout@v3 - name: "Configure iRODS clients" run: | - conda activate "$CONDA_TEST_ENVIRONMENT" - mkdir -p "$HOME/.irods" cat <<'EOF' > "$HOME/.irods/irods_environment.json" { - "irods_host": "localhost", + "irods_host": "irods-server", "irods_port": 1247, "irods_user_name": "irods", "irods_zone_name": "testZone", @@ -88,29 +61,24 @@ jobs: } EOF + # Without this, Actions refuses the connection between the containers: + nc -z -v irods-server 1247 + echo "irods" | script -q -c "iinit" /dev/null ienv ils ilsresc - name: "Build and test baton" + env: + CK_DEFAULT_TIMEOUT: 20 run: | - conda activate "$CONDA_TEST_ENVIRONMENT" - - # Is there an environment variable for this? - CONDA_ENV="/home/runner/conda/envs/$CONDA_TEST_ENVIRONMENT" - - CPPFLAGS="-I$CONDA_ENV/include -I$CONDA_ENV/include/irods" - LDFLAGS="-L$CONDA_ENV/lib -L$CONDA_ENV/lib/irods/externals" - PKG_CONFIG_PATH="$CONDA_ENV/lib/pkgconfig" - - autoreconf -i - ./configure CPPFLAGS="$CPPFLAGS" LDFLAGS="$LDFLAGS" PKG_CONFIG_PATH="$PKG_CONFIG_PATH" - - export LD_LIBRARY_PATH="$CONDA_ENV/lib" + # Avoid git exiting when Actions runs this: + git config --global --add safe.directory "$PWD" - make check - make distcheck DISTCHECK_CONFIGURE_FLAGS="CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\"" + autoreconf -fi + ./configure + make distcheck - name: "Show test log" if: ${{ failure() }} diff --git a/tests/check_baton.c b/tests/check_baton.c index dc7329cc..cf242e0a 100644 --- a/tests/check_baton.c +++ b/tests/check_baton.c @@ -1,6 +1,6 @@ /** - * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 - * Genome Research Ltd. All rights reserved. + * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, + * 2022 Genome Research Ltd. All rights reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -48,8 +48,6 @@ static char *SQL_SETUP_SCRIPT = "scripts/setup_sql.sh"; static char *TEARDOWN_SCRIPT = "scripts/teardown_irods.sh"; static char *SQL_TEARDOWN_SCRIPT = "scripts/teardown_sql.sh"; -static char *BAD_REPLICA_SCRIPT = "scripts/make_bad_replica.sh"; - static void set_current_rods_root(char *in, char *out) { rodsEnv rodsEnv; @@ -2257,9 +2255,7 @@ START_TEST(test_checksum_ignore_stale) { char command[MAX_COMMAND_LEN]; // change checksum of replica 1 without marking stale - snprintf(command, MAX_COMMAND_LEN, "%s/%s -c %s -d %s -a", TEST_ROOT, - BAD_REPLICA_SCRIPT, rods_root, obj_name); - + snprintf(command, MAX_COMMAND_LEN, "iquest --sql setObjectChecksumInvalid %s %s 1", rods_root, obj_name); int ret = system(command); if (ret != 0) raise(SIGTERM); @@ -2269,9 +2265,7 @@ START_TEST(test_checksum_ignore_stale) { ck_assert_int_ne(wrong_checksum_error.code, 0); // mark replica 1 as stale - snprintf(command, MAX_COMMAND_LEN, "%s/%s -c %s -d %s -s", TEST_ROOT, - BAD_REPLICA_SCRIPT, rods_root, obj_name); - + snprintf(command, MAX_COMMAND_LEN, "iquest --sql setObjectReplStale %s %s 1", rods_root, obj_name); ret = system(command); if (ret != 0) raise(SIGTERM); diff --git a/tests/scripts/make_bad_replica.sh b/tests/scripts/make_bad_replica.sh deleted file mode 100755 index b078b830..00000000 --- a/tests/scripts/make_bad_replica.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -usage() { - cat 1>&2 <] - [-d ] - [-r ] - [-i ] - [-s] [-a] [-o] - [-v] - [-h] - -Options - -h Display usage and exit - -v verbose - -c path to collection - -d data object in name - -r replica number to alter - -i irods server container name - -s mark as stale - -a alter checksum - -o corrupt object contents - -EOF -} - -log () { - if [[ $verbose == 1 ]] ; then - printf "${1}" - fi -} - -ils_error() { - echo "Error running ils, object may not exist or iinit may not have been run" - exit 1 -} - -#Default values -repl_num=1 -irods_container="irods-server" - -while getopts "hvc:d:r:i:sao" option; do - case "$option" in - h) - usage - exit 0 - ;; - v) - verbose=1 - ;; - c) - coll="$OPTARG" - ;; - d) - obj="$OPTARG" - ;; - r) - repl_num="$OPTARG" - ;; - i) - irods_container="$OPTARG" - ;; - s) - stale=1 - ;; - a) - check=1 - ;; - o) - corrupt=1 - ;; - - esac -done - -if [[ -z $coll ]] ; then - printf "Collection path required" - usage - exit 1 -elif [[ -z $obj ]] ; then - printf "Object name required" - usage - exit 1 -else - exists=$(ils "$coll/$obj") || ils_error -fi - -log "Object: $coll/$obj \nReplica: $repl_num \nOperations: \n" -updates="" - -if [[ -n $stale ]] ; then - log " - make stale \n" - updates+="DATA_IS_DIRTY='0' " - op=1 -fi - -if [[ -n $check ]] ; then - log " - alter checksum \n" - updates+="DATA_CHECKSUM='0' " - op=1 -fi - -if [[ -n $corrupt ]] ; then - log " - corrupt data \n" - fs_path=$(ils -L $coll/$obj | grep /var/lib/irods | sed "$(($repl_num+1))q;d" | sed -e 's/.*generic //g') - docker exec -u root "$irods_container" bash -c "echo 'This file is corrupted' > $fs_path" || exit 1; - op=2 -fi - -if [[ -z $op ]]; then - printf "No operation requested \n" - exit 0 -fi - - -if [[ $op -eq 1 ]]; then - docker exec -u root "$irods_container" sudo -u irods psql -d ICAT -c "UPDATE r_data_main d SET $updates FROM r_coll_main c WHERE d.coll_id = c.coll_id AND d.DATA_NAME='$obj' AND c.COLL_NAME='$coll' AND d.DATA_REPL_NUM='$repl_num';" || exit 1 -fi - - diff --git a/tests/scripts/multi_write.sh b/tests/scripts/multi_write.sh deleted file mode 100755 index 8bc3f7d7..00000000 --- a/tests/scripts/multi_write.sh +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env bash - -if [ "$1" == "" ]; then - echo "Please provide an irods collection path to put the files into,\ - it is suggested that this collection path does not currently exist to make it easier to delete once the test is complete" && exit 1 -fi - -# produce temp file -TMPFILE=$(mktemp) # TMPDIR can be set if the file should be placed elsewhere -dd if=/dev/zero of=$TMPFILE bs=1M count=10 > /dev/null 2>&1 - -# make collection -imkdir -p $1 - -i=0 -while [ $i -lt 10 ] ; do - - json=$(jq -n "{collection: \"$1\", data_object: \"$(basename $TMPFILE)_$i.txt\", directory: \"$(dirname $TMPFILE)\", file: \"$(basename $TMPFILE)\"}") - - echo $json | baton-put > /dev/null 2>&1 & echo $json | baton-put > /dev/null 2>&1 - - ((i++)) -done - -# show files to demonstrate which have zero size -ils -l $1 || (echo "could not run ils, please check that you have initialised iRODS" && exit 1) diff --git a/tests/sql/specific_queries.sql b/tests/sql/specific_queries.sql index b6ac6502..bf464447 100644 --- a/tests/sql/specific_queries.sql +++ b/tests/sql/specific_queries.sql @@ -2,3 +2,5 @@ 'SELECT Collection.coll_name AS coll_name, Data.data_name AS data_name, Data.data_id AS data_id, Data.data_repl_num AS data_repl_num, Data.data_checksum AS data_checksum, Data.modify_ts AS data_modify_ts, Data.data_is_dirty AS data_repl_status FROM R_DATA_MAIN Data INNER JOIN R_COLL_MAIN Collection ON Data.coll_id = Collection.coll_id WHERE CAST(Data.modify_ts AS INT) > CAST(? AS INT) AND CAST(Data.modify_ts AS INT) <= CAST(? AS INT)' dataModifiedPartial 'SELECT DISTINCT Data.data_id AS data_id FROM R_DATA_MAIN Data INNER JOIN R_OBJT_METAMAP MetadataMap ON Data.data_id = MetadataMap.object_id WHERE CAST(MetadataMap.modify_ts AS INT) > CAST(? AS INT) AND CAST(MetadataMap.modify_ts AS INT) <= CAST(? AS INT)' metadataModifiedIdOnly 'SELECT Collection.coll_name AS coll_name, Data.data_name AS data_name, Data.data_id AS data_id, Metadata.meta_id AS meta_id, Metadata.meta_attr_name AS meta_attr_name, Metadata.meta_attr_value AS meta_attr_value, Metadata.meta_attr_unit AS meta_attr_unit, MetadataMap.modify_ts AS meta_modify_ts FROM R_DATA_MAIN Data INNER JOIN R_COLL_MAIN Collection ON Data.coll_id = Collection.coll_id INNER JOIN R_OBJT_METAMAP MetadataMap ON Data.data_id = MetadataMap.object_id INNER JOIN R_META_MAIN Metadata ON MetadataMap.meta_id = Metadata.meta_id WHERE CAST(MetadataMap.modify_ts AS INT) > CAST(? AS INT) AND CAST(MetadataMap.modify_ts AS INT) <= CAST(? AS INT)' metadataModifiedPartial +'UPDATE r_data_main dm SET DATA_IS_DIRTY = 0 FROM r_coll_main cm WHERE dm.coll_id = cm.coll_id AND cm.COLL_NAME = ? AND dm.DATA_NAME= ? AND dm.DATA_REPL_NUM= ?' setObjectReplStale +'UPDATE r_data_main dm SET DATA_CHECKSUM = 0 FROM r_coll_main cm WHERE dm.coll_id = cm.coll_id AND cm.COLL_NAME = ? AND dm.DATA_NAME= ? AND dm.DATA_REPL_NUM= ?' setObjectChecksumInvalid From 01195e2fb7e763f7c81098be7913a530dff06f29 Mon Sep 17 00:00:00 2001 From: mksanger Date: Thu, 16 Mar 2023 14:40:42 +0000 Subject: [PATCH 5/7] Use github hosted images for docker compose and ci --- .github/workflows/unit-tests.yml | 12 ++++++------ docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 722127ee..433a6886 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -13,18 +13,18 @@ jobs: include: # iRODS 4.2.11 clients on bionic - irods: "4.2.11" - build_image: "wsinpg/ub-18.04-irods-clients-dev-4.2.11:latest" - server_image: "wsinpg/ub-18.04-irods-4.2.11:latest" + build_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-dev-4.2.11:latest" + server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest" experimental: false # iRODS 4.3.0 clients on bionic - irods: "4.3.0" - build_image: "wsinpg/ub-18.04-irods-clients-dev-4.3.0:latest" - server_image: "wsinpg/ub-18.04-irods-4.3.0:latest" + build_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-dev-4.3.0:latest" + server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.3.0:latest" experimental: true # iRODS 4.3.0 clients on focal - irods: "4.3.0" - build_image: "wsinpg/ub-20.04-irods-clients-dev-4.3.0:latest" - server_image: "wsinpg/ub-18.04-irods-4.3.0:latest" + build_image: "ghcr.io/wtsi-npg/ub-20.04-irods-clients-dev-4.3.0:latest" + server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.3.0:latest" experimental: true container: diff --git a/docker-compose.yml b/docker-compose.yml index 4d365706..0d1621db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: irods-server: container_name: irods-server - image: "wsinpg/ub-18.04-irods-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}" + image: "ghcr.io/wtsi-npg/ub-18.04-irods-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}" restart: always ports: - "1247:1247" From 2152face04c6a36d8f7ef6e19d5cb8f367f34359 Mon Sep 17 00:00:00 2001 From: Keith James Date: Mon, 27 Mar 2023 16:26:26 +0100 Subject: [PATCH 6/7] Fix missing format string in logging --- src/baton.c | 14 +++++++------- src/json_query.c | 11 +++++------ src/list.c | 8 ++++---- tests/check_baton.c | 31 ++++++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/baton.c b/src/baton.c index ac271721..5bb2519a 100644 --- a/src/baton.c +++ b/src/baton.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2020 Genome Research + * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2020, 2023 Genome Research * Ltd. All rights reserved. * * This program is free software: you can redistribute it and/or modify @@ -560,7 +560,7 @@ json_t *search_metadata(rcComm_t *conn, json_t *query, char *zone_name, return results; error: - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); if (results) json_decref(results); if (collections) json_decref(collections); @@ -589,7 +589,7 @@ json_t *search_specific(rcComm_t *conn, json_t *query, char *zone_name, return results; error: - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); if (results) json_decref(results); @@ -658,11 +658,11 @@ int modify_permissions(rcComm_t *conn, rodsPath_t *rods_path, error: if (conn->rError) { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); log_rods_errstack(ERROR, conn->rError); } else { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); } return error->code; @@ -775,11 +775,11 @@ int modify_metadata(rcComm_t *conn, rodsPath_t *rods_path, error: if (conn->rError) { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); log_rods_errstack(ERROR, conn->rError); } else { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); } return error->code; diff --git a/src/json_query.c b/src/json_query.c index 44d08e5d..17168ee3 100644 --- a/src/json_query.c +++ b/src/json_query.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013, 2014, 2015, 2016, 2019, 2021 Genome Research + * Copyright (C) 2013, 2014, 2015, 2016, 2019, 2021, 2023 Genome Research * Ltd. All rights reserved. * * This program is free software: you can redistribute it and/or modify @@ -473,11 +473,11 @@ json_t *do_query(rcComm_t *conn, genQueryInp_t *query_in, error: if (conn->rError) { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); log_rods_errstack(ERROR, conn->rError); } else { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); } if (query_out) free_query_output(query_out); @@ -572,11 +572,11 @@ json_t *do_squery(rcComm_t *conn, specificQueryInp_t *squery_in, error: if (conn->rError) { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); log_rods_errstack(ERROR, conn->rError); } else { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); } if (query_out) free_query_output(query_out); @@ -1369,4 +1369,3 @@ json_t *revmap_replicate_results(rcComm_t *conn, json_t *results, return NULL; } - diff --git a/src/list.c b/src/list.c index 1e751fbe..ed1fb640 100644 --- a/src/list.c +++ b/src/list.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2019, 2021 Genome + * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2019, 2021, 2023 Genome * Research Ltd. All rights reserved. * * This program is free software: you can redistribute it and/or modify @@ -171,11 +171,11 @@ static json_t *list_collection(rcComm_t *conn, rodsPath_t *rods_path, error: if (conn->rError) { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); log_rods_errstack(ERROR, conn->rError); } else { - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); } return NULL; @@ -354,7 +354,7 @@ json_t *list_path(rcComm_t *conn, rodsPath_t *rods_path, option_flags flags, error: if (result) json_decref(result); - logmsg(ERROR, error->message); + logmsg(ERROR, "%s", error->message); return NULL; } diff --git a/tests/check_baton.c b/tests/check_baton.c index cf242e0a..ed35f209 100644 --- a/tests/check_baton.c +++ b/tests/check_baton.c @@ -1,6 +1,6 @@ /** * Copyright (C) 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, - * 2022 Genome Research Ltd. All rights reserved. + * 2022, 2023 Genome Research Ltd. All rights reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -2868,6 +2868,34 @@ START_TEST(test_regression_github_issue252) { } END_TEST +START_TEST(test_regression_github_issue267) { + option_flags flags = 0; + rodsEnv env; + rcComm_t *conn = rods_login(&env); + + char rods_root[MAX_PATH_LEN]; + set_current_rods_root(TEST_COLL, rods_root); + + // A data object named literally "%s.txt" (but which doesn't exist) + char* obj_path =" /testZone/%s.txt"; + + rodsPath_t rods_obj_path; + baton_error_t resolve_error; + resolve_rods_path(conn, &env, &rods_obj_path, obj_path, + flags, &resolve_error); + ck_assert_int_eq(resolve_error.code, -1); + + flags = SEARCH_OBJECTS; + + baton_error_t error; + // This segfaulted before being fixed + json_t *results = list_path(conn, &rods_obj_path, flags, &error); + ck_assert_int_eq(error.code, -317000); // USER_INPUT_PATH_ERR + + if (conn) rcDisconnect(conn); +} +END_TEST + Suite *baton_suite(void) { Suite *suite = suite_create("baton"); @@ -2989,6 +3017,7 @@ Suite *baton_suite(void) { tcase_add_test(regression, test_regression_github_issue140); tcase_add_test(regression, test_regression_github_issue242); tcase_add_test(regression, test_regression_github_issue252); + tcase_add_test(regression, test_regression_github_issue267); suite_add_tcase(suite, utilities); suite_add_tcase(suite, basic); From 314f9cf0dec2c07380271b011e5564e402cdaa39 Mon Sep 17 00:00:00 2001 From: Keith James Date: Tue, 4 Apr 2023 10:51:35 +0100 Subject: [PATCH 7/7] Update changelog for 4.0.1 release --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 78a1a8ca..be029a7d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ [Upcoming] + [4.0.1] + + Fix missing format string in logging which caused a segfault when + log message arugments contained a printf placeholder. + [4.0.0] Improve connection management by closing the connection while