[QOLDEV-1013] update test config

- Ensure shell variables are escaped - Add reporting-only testing on CKAN 2.11 and latest master - Prepare scenario tests for CKAN 2.11
qld-gov-au · Jan 2, 2025 · c98cf45 · c98cf45
1 parent f3e1cbb
commit c98cf45
Show file tree

Hide file tree

Showing 12 changed files with 82 additions and 93 deletions.
diff --git a/.ahoy.yml b/.ahoy.yml
@@ -38,7 +38,7 @@ commands:
       ahoy title "Building and starting Docker containers"
       sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
-      ahoy cli '$APP_DIR/bin/init.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh'
       echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
       if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
@@ -66,7 +66,9 @@ commands:
 
   logs:
     usage: Show Docker logs.
-    cmd: sh bin/docker-compose.sh logs "$@"
+    cmd: |
+        ahoy title "Output logs"
+        sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
@@ -77,9 +79,9 @@ commands:
     cmd: |
       CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
       if [ "${#}" -ne 0 \]; then
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*"
       else
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh'
       fi
 
   doctor:
@@ -90,7 +92,7 @@ commands:
     usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -126,13 +128,13 @@ commands:
     cmd: |
       docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
       docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI'
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
       ahoy title 'Run unit tests'
-      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} "${APP_DIR}"/ckanext --junit-xml=test/junit/results.xml' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
@@ -142,7 +144,8 @@ commands:
       ahoy cli "rm -f test/screenshots/*"
       ahoy start-ckan-job-worker
       sleep 5
-      ahoy cli "behave -k ${*:-test/features} $EXTRA_TAGS --tags=${BEHAVE_TAG:--format_autocomplete}" || \
+      JUNIT_OUTPUT="--junit --junit-directory=test/junit/"
+      ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} $EXTRA_TAGS --tags=${BEHAVE_TAG:--format_autocomplete}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-ckan-job-worker
 

diff --git a/.docker/test.ini b/.docker/test.ini
@@ -1,16 +1,3 @@
-#
-# CKAN - Pylons configuration
-#
-# These are some of the configuration options available for your CKAN
-# instance. Check the documentation in 'doc/configuration.rst' or at the
-# following URL for a description of what they do and the full list of
-# available options:
-#
-# http://docs.ckan.org/en/latest/maintaining/configuration.html
-#
-# The %(here)s variable will be replaced with the parent directory of this file
-#
-
 [DEFAULT]
 debug = false
 smtp_server = localhost:8025
@@ -32,9 +19,7 @@ full_stack = true
 cache_dir = /tmp/%(ckan.site_id)s/
 beaker.session.key = ckan
 
-# This is the secret token that the beaker library uses to hash the cookie sent
-# to the client. `paster make-config` generates a unique value for this each
-# time it generates a config file.
+SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE
 beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
 
 # `paster make-config` generates a unique value for this each time it generates
@@ -82,10 +67,6 @@ ckan.redis.url = redis://redis:6379
 
 ## Plugins Settings
 
-# Note: Add ``datastore`` to enable the CKAN DataStore
-#       Add ``datapusher`` to enable DataPusher
-#       Add ``resource_proxy`` to enable resource proxying and get around the
-#       same origin policy
 ckan.plugins =
     validation
     scheming_datasets

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -12,57 +12,76 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
-        timeout-minutes: 2
-      - uses: actions/setup-python@v4
-        timeout-minutes: 5
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install requirements
-        timeout-minutes: 5
         run: pip install flake8 pycodestyle
       - name: Check syntax
-        timeout-minutes: 5
         run: flake8
 
   test:
     needs: lint
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9]
+        ckan-version: ["2.10", "2.9"]
+        experimental: [false]
+        include:
+          - ckan-version: "2.11"
+            experimental: true
+          - ckan-version: "master"
+            experimental: true  # master is unstable, good to know if we are compatible or not
 
-    name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
+    name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder:23.7.0
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v3
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 2
 
       - name: Build
+        continue-on-error: ${{ matrix.experimental }}
         run: bin/build.sh
         timeout-minutes: 15
 
       - name: Test
+        continue-on-error: ${{ matrix.experimental }}
         run: bin/test.sh
         timeout-minutes: 20
 
       - name: Retrieve logs
         if: always()
         run: ahoy logs
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
 
-      - name: Retrieve screenshots
+      - name: Retrieve results
         if: always()
         run: bin/process-artifacts.sh
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
 
+      - name: Test Summary
+        uses: test-summary/action@v2
+        continue-on-error: ${{ matrix.experimental }}
+        with:
+          paths: "/tmp/artifacts/junit/*.xml"
+        if: always()
+
       - name: Upload screenshots
-        if: failure()
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
+        continue-on-error: ${{ matrix.experimental }}
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh
@@ -9,7 +9,7 @@ CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
 CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
 CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
@@ -19,6 +19,10 @@ add_user_if_needed () {
         password="${4:-Password123!}"
 }
 
+api_call () {
+    wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2
+}
+
 add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
 ckan_cli sysadmin add "${CKAN_USER_NAME}"
 
@@ -44,35 +48,26 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
-    curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
-        "description": "Organisation for testing issues"}' \
-    ${CKAN_ACTION_URL}/organization_create
+    api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
+        "description": "Organisation for testing issues"}' organization_create
 )
 
-TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
+TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON "${APP_DIR}"/bin/extract-id.py)
 
 echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' member_create
+
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' member_create
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' member_create
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \
-    ${CKAN_ACTION_URL}/member_create
 ##
 # END.
 #
 
 # Creating basic test data which has datasets with resources
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "warandpeace", "owner_org": "'"${TEST_ORG_ID}"'",
-"author_email": "admin@localhost", "license_id": "other-open", "notes": "test"}' \
-    ${CKAN_ACTION_URL}/package_create
+api_call '{"name": "warandpeace", "owner_org": "'"${TEST_ORG_ID}"'",
+"author_email": "admin@localhost", "license_id": "other-open", "notes": "test"}' package_create
 
-. ${APP_DIR}/bin/deactivate
+. "${APP_DIR}"/bin/deactivate
diff --git a/bin/docker-compose.sh b/bin/docker-compose.sh
@@ -1,13 +1,15 @@
 #!/bin/sh
 
+set -x
+
 # Pass commands to Docker Compose v1 or v2 depending on what is present
 
-if (which docker-compose >/dev/null); then
-    # Docker Compose v1
-    docker-compose $*
-elif (docker compose ls >/dev/null); then
+if (docker compose ls >/dev/null); then
     # Docker Compose v2
     docker compose $*
+elif (which docker-compose >/dev/null); then
+    # Docker Compose v1
+    docker-compose $*
 else
     # Docker Compose not found
     exit 1

diff --git a/bin/init-ext.sh b/bin/init-ext.sh
@@ -32,8 +32,7 @@ install_requirements () {
     done
 }
 
-. ${APP_DIR}/bin/activate
-
+. "${APP_DIR}"/bin/activate
 install_requirements . dev-requirements requirements-dev
 for extension in . `ls -d $SRC_DIR/ckanext-*`; do
     install_requirements $extension requirements pip-requirements
@@ -46,4 +45,4 @@ installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1
 # Validate that the extension was installed correctly.
 if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
 
-. ${APP_DIR}/bin/deactivate
+. "${APP_DIR}"/bin/deactivate
diff --git a/bin/init.sh b/bin/init.sh
@@ -4,7 +4,7 @@
 #
 set -e
 
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init
 

diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh
@@ -6,8 +6,9 @@ set -e
 
 # Create screenshots directory in case it was not created before. This is to
 # avoid this script to fail when copying artifacts.
-ahoy cli "mkdir -p test/screenshots"
+ahoy cli "mkdir -p test/screenshots test/junit"
 
 # Copy from the app container to the build host for storage.
-mkdir -p /tmp/artifacts/behave
+mkdir -p /tmp/artifacts/behave /tmp/artifacts/junit
 docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/
+docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/junit /tmp/artifacts/
diff --git a/bin/serve.sh b/bin/serve.sh
@@ -1,23 +1,9 @@
 #!/usr/bin/env sh
 set -e
 
-dockerize -wait tcp://postgres:5432 -timeout 1m
-dockerize -wait tcp://solr:8983 -timeout 1m
-dockerize -wait tcp://redis:6379 -timeout 1m
-
-for i in `seq 1 60`; do
-    if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then
-        echo "Database became ready on attempt $i"
-        break
-    else
-        echo "Database not yet ready, retrying (attempt $i)..."
-        sleep 1
-    fi
-done
-
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 if (which ckan > /dev/null); then
-    ckan -c ${CKAN_INI} run -r -t
+    ckan -c ${CKAN_INI} run --disable-reloader --threaded
 else
     paster serve ${CKAN_INI}
 fi
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,6 +1,6 @@
 beautifulsoup4==4.9.1
-behave==1.2.6
-behaving==1.5.6
+behaving==3.1.5
+Appium-Python-Client==2.10.1
 ckanapi==4.3
 flake8==3.8.3
 mock

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,6 +27,8 @@ services:
         condition: service_healthy
       solr:
         condition: service_started
+      redis:
+        condition: service_started
     networks:
       - amazeeio-network
       - default

diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py
@@ -10,7 +10,7 @@ def debug_screenshot(context):
     """
     if context.persona and context.persona.get('debug') == 'True':
         context.execute_steps(u"""
-            Then I take a screenshot
+            When I take a screenshot
         """)
 
 
@@ -47,7 +47,7 @@ def log_in(context):
 @when(u'I expand the browser height')
 def expand_height(context):
     # Work around x=null bug in Selenium set_window_size
-    context.browser.driver.set_window_rect(x=0, y=0, width=1024, height=4096)
+    context.browser.driver.set_window_rect(x=0, y=0, width=1024, height=3072)
 
 
 @when(u'I log in directly')
@@ -58,10 +58,11 @@ def log_in_directly(context):
     :return:
     """
 
-    assert context.persona, "A persona is required to log in, found [{}] in context. Have you configured the personas in before_scenario?".format(context.persona)
+    assert context.persona, "A persona is required to log in, found [{}] in context." \
+        " Have you configured the personas in before_scenario?".format(context.persona)
     context.execute_steps(u"""
         When I attempt to log in with password "$password"
-        Then I should see an element with xpath "//a[@title='Log out']"
+        Then I should see an element with xpath "//*[@title='Log out' or @data-bs-title='Log out']/i[contains(@class, 'fa-sign-out')]"
     """)