From 4c31aef5ac8ca9738f557eaba3d43efc21094412 Mon Sep 17 00:00:00 2001 From: Aaron Elkiss Date: Thu, 23 May 2024 16:47:27 -0400 Subject: [PATCH] Use healthcheck instead of wait-for Also, remove extra docker-compose.test.yml --- .github/workflows/tests.yml | 4 +- bin/test_and_cover.sh | 4 -- bin/wait-for | 118 ------------------------------------ docker-compose.test.yml | 50 --------------- docker-compose.yml | 65 +++++++++++++++++--- 5 files changed, 59 insertions(+), 182 deletions(-) delete mode 100755 bin/test_and_cover.sh delete mode 100755 bin/wait-for delete mode 100644 docker-compose.test.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f90682cf..b9ce44e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,9 +17,9 @@ jobs: - uses: actions/checkout@v4 - name: Build docker image - run: docker compose --file docker-compose.test.yml build + run: docker compose build - name: Run tests - run: docker compose --file docker-compose.test.yml run sut + run: docker compose run test-and-cover env: GITHUB_TOKEN: ${{ secrets.github_token }} diff --git a/bin/test_and_cover.sh b/bin/test_and_cover.sh deleted file mode 100755 index e69ea0a5..00000000 --- a/bin/test_and_cover.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -bin/wait-for --timeout=300 mariadb:3306 minio:9000 pushgateway:9091 rabbitmq:5672 -cover -test -report Coveralls -make 'prove; exit $?' diff --git a/bin/wait-for b/bin/wait-for deleted file mode 100755 index 9fd63d83..00000000 --- a/bin/wait-for +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -# From https://github.com/eficode/wait-for - -# The MIT License (MIT) -# -# Copyright (c) 2017 Eficode Oy -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -TIMEOUT=15 -QUIET=0 - -echoerr() { - if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi -} - -usage() { - exitcode="$1" - cat << USAGE >&2 -Usage: - $cmdname host:port [host:port ...] [-t timeout] [-- command args] - -q | --quiet Do not output any status messages - -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit "$exitcode" -} - -wait_for() { - command="$*" - for i in `seq $TIMEOUT` ; do - - result=0 - - for dep in $DEPENDENCIES; do - host=$(printf "%s\n" "$dep"| cut -d : -f 1) - port=$(printf "%s\n" "$dep"| cut -d : -f 2) - if [ "$host" = "" -o "$port" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 - fi - nc -z "$host" "$port" > /dev/null 2>&1 - nc_result=$? - if [ $nc_result -ne 0 ] ; then - result=1 - fi - done - - if [ $result -eq 0 ] ; then - if [ -n "$command" ] ; then - exec $command - fi - exit 0 - fi - sleep 1 - done - echo "Operation timed out" >&2 - exit 1 -} - -DEPENDENCIES="" -while [ $# -gt 0 ] -do - case "$1" in - *:* ) - DEPENDENCIES+=" $1" - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -t) - TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - break - ;; - --help) - usage 0 - ;; - *) - echoerr "Unknown argument: $1" - usage 1 - ;; - esac -done - -if [ "${#DEPENDENCIES[@]}" -eq "0" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 -fi - -wait_for "$@" diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 3ae9c7f0..00000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: '3' - -services: - - sut: - build: . - environment: - - HTFEED_CONFIG=/usr/local/feed/etc/config_test.yml - - PUSHGATEWAY=http://pushgateway:9091 - - FEED_HOME=/usr/local/feed - - TEST=1 - - VERSION=feed-testing - # pass through info needed by coveralls uploader - - GITHUB_TOKEN - - GITHUB_RUN_ID - - GITHUB_EVENT_TOKEN - - GITHUB_EVENT_PATH - - GITHUB_SHA - - GITHUB_REF - - GITHUB_ACTIONS - command: bin/test_and_cover.sh - depends_on: - - mariadb - - minio - - pushgateway - - rabbitmq - - mariadb: - image: ghcr.io/hathitrust/db-image:latest - volumes: - - ./etc/ingest.sql:/docker-entrypoint-initdb.d/999-ingest.sql - restart: always - environment: - MYSQL_ROOT_PASSWORD: mysqlroot - - minio: - image: minio/minio - restart: always - environment: - MINIO_ACCESS_KEY: TESTINGACCESSKEY - MINIO_SECRET_KEY: testingsecretkey - command: server /data - - pushgateway: - image: prom/pushgateway - command: - - --web.enable-admin-api - - rabbitmq: - image: rabbitmq diff --git a/docker-compose.yml b/docker-compose.yml index fe351497..d294c715 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,13 @@ -version: '3' +--- + +x-condition-healthy: &healthy + condition: service_healthy + +x-healthcheck-defaults: &healthcheck-defaults + interval: 5s + timeout: 10s + start_period: 10s + retries: 5 services: @@ -10,15 +19,43 @@ services: environment: - HTFEED_CONFIG=/usr/local/feed/etc/config_test.yml - FEED_HOME=/usr/local/feed + - VERSION=feed-development + - PUSHGATEWAY=http://pushgateway:9091 - TEST=1 + command: prove + depends_on: + mariadb: *healthy + minio: *healthy + pushgateway: *healthy + rabbitmq: *healthy + + # Note: for permissions purposes this does NOT bind in the local development + # environment, so local changes after running docker compose build will NOT + # be reflected. + test-and-cover: + build: . + volumes: + - ./clamav:/var/lib/clamav + environment: + - HTFEED_CONFIG=/usr/local/feed/etc/config_test.yml + - FEED_HOME=/usr/local/feed - VERSION=feed-development - PUSHGATEWAY=http://pushgateway:9091 - command: bin/wait-for --timeout=300 mariadb:3306 minio:9000 pushgateway:9091 rabbitmq:5672 -- prove + - TEST=1 + # pass through info needed by coveralls uploader + - GITHUB_TOKEN + - GITHUB_RUN_ID + - GITHUB_EVENT_TOKEN + - GITHUB_EVENT_PATH + - GITHUB_SHA + - GITHUB_REF + - GITHUB_ACTIONS + command: cover -test -report Coveralls -make 'prove; exit $?' depends_on: - - mariadb - - minio - - pushgateway - - rabbitmq + mariadb: *healthy + minio: *healthy + pushgateway: *healthy + rabbitmq: *healthy ingest: build: . @@ -40,7 +77,7 @@ services: - FEED_HOME=/usr/local/feed - VERSION=feed-development depends_on: - - mariadb + mariadb: *healthy validate: build: . @@ -53,7 +90,7 @@ services: - VERSION=feed-development command: bash -c "/bin/ls /tmp/stage/toingest/test/*.zip | xargs -n 1 basename | sed s/.zip// | perl -w /usr/local/feed/bin/validate_volume.pl -p simple -n test --no-clean" depends_on: - - mariadb + mariadb: *healthy mariadb: image: ghcr.io/hathitrust/db-image:latest @@ -62,6 +99,9 @@ services: restart: always environment: MYSQL_ROOT_PASSWORD: mysqlroot + healthcheck: + <<: *healthcheck-defaults + test: ["CMD", "healthcheck.sh", "--su-mysql", "--connect", "--innodb_initialized"] # S3 compatible object storage minio: @@ -71,6 +111,9 @@ services: MINIO_ACCESS_KEY: TESTINGACCESSKEY MINIO_SECRET_KEY: testingsecretkey command: server /data + healthcheck: + <<: *healthcheck-defaults + test: [ "CMD", "/usr/bin/curl", "-s", "-f", "http://localhost:9000/minio/health/live"] pushgateway: image: prom/pushgateway @@ -78,9 +121,15 @@ services: - --web.enable-admin-api ports: - "9091:9091" + healthcheck: + <<: *healthcheck-defaults + test: [ "CMD", "wget", "--quiet", "--tries=1", "-O", "/dev/null", "pushgateway:9091/-/healthy" ] rabbitmq: image: rabbitmq + healthcheck: + <<: *healthcheck-defaults + test: ["CMD", "rabbitmqctl", "status"] volumes: repository_link: