feat: add a more robust set of tests (#85)

* wip: initial tests refactor

* feat: d10 and d11 tests

* tests: d10 remove corepack and use d11 yarn install approach

* refactor: tests rename

* refactor: moves common code into _common.bash

* feat: require-dev on d11 test as well

* feat: test drupal version with drush

* feat: add bats_require_minimum_version 1.5.0

* feat: single full.bats test and GHA matrix strategy

* fix: using eval for running bats tests with env vars

* refactor:  move common code back to its own file

* fix: avoid setting DRUPAL_CORE on default tests

* fix: allow to run tests without TEST_DRUPAL_CORE

It will run the default core

* feat: test for presence of web/core and vendor

* fix: allow drupal core version to work without a specified test core

* refactor: fix typos and composer.json cleanup

* docs: README tweaks about testing

* feat: using bats-assert

* fix: use `brew --prefix` for BATS_LIB_PATH

* feat: using submodules for bats

* docs: small tweak

* docs: proper submodule git commands

* docs: simplified tests commands

* docs: apply suggestions from code review

Co-authored-by: tyler36 <[email protected]>

---------

Co-authored-by: tyler36 <[email protected]>
Co-authored-by: Moshe Weitzman <[email protected]>

.github/workflows/tests.yml

@@ -28,15 +28,22 @@ jobs:
     strategy:
       matrix:
         ddev_version: [stable, HEAD]
+        drupal_version: ["default", "10", "11"]
       fail-fast: false
 
     runs-on: ubuntu-latest
 
     steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
     - uses: ddev/github-action-add-on-test@v2
       with:
+        disable_checkout_action: true
         ddev_version: ${{ matrix.ddev_version }}
         token: ${{ secrets.GITHUB_TOKEN }}
         debug_enabled: ${{ github.event.inputs.debug_enabled }}
         addon_repository: ${{ env.GITHUB_REPOSITORY }}
         addon_ref: ${{ env.GITHUB_REF }}
+        test_command: eval TEST_DRUPAL_CORE=${{ matrix.drupal_version }} ./tests/bats/bin/bats tests

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "tests/bats"]
+	path = tests/bats
+	url = https://github.com/bats-core/bats-core.git
+[submodule "tests/helpers/bats-assert"]
+	path = tests/helpers/bats-assert
+	url = https://github.com/bats-core/bats-assert.git
+[submodule "tests/helpers/bats-support"]
+	path = tests/helpers/bats-support
+	url = https://github.com/bats-core/bats-support.git

README.md

@@ -109,6 +109,39 @@ ddev phpcbf -q
 
 3. Mark the file as executable: `chmod +x pre-commit`.
 
+## Add-on tests
+
+Tests are done with Bats. It is a testing framework that uses Bash. 
+
+To run tests locally you need to first install bats' git submodules with: 
+
+```sh
+git submodule update --init
+```
+
+Then you can run within the root of this project:
+
+```sh
+./tests/bats/bin/bats ./tests
+```
+
+Tests will be run using the default drupal core of the contrib. To test against a different Drupal core version, update the `TEST_DRUPAL_CORE` environment 
+variable.
+
+i.e. `TEST_DRUPAL_CORE=11 ./tests/bats/bin/bats ./tests`.
+
+Tests are triggered automatically on every push to the 
+repository, and periodically each night. The automated tests are agains all of
+the supported Drupal core versions.
+
+Please make sure to attend to test failures when they happen. Others will be 
+depending on you. 
+
+Also, consider adding tests to test for bugs or new features on your PR.
+
+To learn more about Bats see the [documentation][bats-docs].
+
+[bats-docs]: https://bats-core.readthedocs.io/en/stable/
 
 ## Troubleshooting
 

tests/_common.bash

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+_common_setup() {
+  bats_require_minimum_version 1.5.0
+  export DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )/.."
+  export PROJNAME=test-drupal-contrib
+  export TESTDIR=~/tmp/${PROJNAME}
+  mkdir -p $TESTDIR
+  export DDEV_NON_INTERACTIVE=true
+  ddev delete -Oy ${PROJNAME} >/dev/null 2>&1 || true
+  cp -R ${DIR}/tests/testdata/test_drupal_contrib/* ${TESTDIR}
+  cd ${TESTDIR}
+  ddev config  --project-name=${PROJNAME} --project-type=drupal --docroot=web
+  if [ -n "$TEST_DRUPAL_CORE" ] && [ "$TEST_DRUPAL_CORE" != "default" ]; then
+    echo -e "web_environment:\n    - DRUPAL_CORE=^${TEST_DRUPAL_CORE}" > .ddev/config.~overrides.yaml
+  fi
+  ddev get ${DIR}
+}
+
+_common_teardown() {
+  ddev delete -Oy ${PROJNAME} >/dev/null 2>&1
+  [ "${TESTDIR}" != "" ] && rm -rf ${TESTDIR}
+}
+
+_common_test_poser() {
+  ddev poser
+  ddev mutagen sync
+  ls -la web/core
+  ls -la vendor/
+  ls -la
+}

tests/default-no-composer-json.bats

@@ -0,0 +1,19 @@
+setup_file() {
+  if [ -n "$TEST_DRUPAL_CORE" ] && [ "$TEST_DRUPAL_CORE" != "default" ]; then
+    skip "TEST_DRUPAL_CORE=$TEST_DRUPAL_CORE not handled by this test suite" >&2
+  fi
+  load '_common.bash'
+  _common_setup
+  ddev start
+}
+
+teardown_file() {
+  load '_common.bash'
+  _common_teardown
+}
+
+@test "ddev poser without composer.json" {
+  load '_common.bash'
+  rm -f composer.json
+  _common_test_poser
+}

tests/full.bats

@@ -0,0 +1,55 @@
+load helpers/bats-support/load.bash
+load helpers/bats-assert/load.bash
+
+setup_file() {
+  if [ -n "$TEST_DRUPAL_CORE" ] && [ "$TEST_DRUPAL_CORE" != "10" ] && [ "$TEST_DRUPAL_CORE" != "11"  ]; then
+    skip "TEST_DRUPAL_CORE=$TEST_DRUPAL_CORE not handled by this test suite" >&2
+  fi
+  load '_common.bash'
+  _common_setup
+  ddev config --php-version=8.2
+  if [ "$TEST_DRUPAL_CORE" = "11" ]; then
+    ddev config --php-version=8.3 --corepack-enable
+  fi
+  ddev start
+}
+
+teardown_file() {
+  load '_common.bash'
+  _common_teardown
+}
+
+@test "ddev poser with composer.json" {
+  load '_common.bash'
+  _common_test_poser
+}
+
+@test "ddev symlink-project" {
+  ddev symlink-project
+  ddev mutagen sync
+  ls -la web/modules/custom/test_drupal_contrib/test_drupal_contrib.info.yml
+}
+
+@test "php tools availability" {
+  ddev phpcs --version
+  ddev phpstan --version
+  ddev phpunit --version
+}
+
+@test "drupal core version" {
+  run -0 ddev exec 'drush st --fields=drupal-version --format=string | cut -d. -f1'
+  if [ -n "${TEST_DRUPAL_CORE}" ]; then
+    assert_output "${TEST_DRUPAL_CORE}"
+  else
+    DDEV_DRUPAL_CORE=$(ddev exec 'echo "${DRUPAL_CORE/^/}"')
+    assert_output "$DDEV_DRUPAL_CORE"
+  fi
+}
+
+@test "node tools availability" {
+  ddev exec "cd web/core && yarn install"
+  ddev exec touch web/core/.env
+  ddev mutagen sync
+  ddev stylelint --version
+  ddev eslint --version
+}

tests/testdata/test_drupal_contrib/composer.json

@@ -0,0 +1,47 @@
+{
+    "name": "drupal/ddev_drupal_contrib",
+    "type": "drupal-module",
+    "license": "GPL-2.0+",
+    "homepage": "https://github.com/ddev/ddev-drupal-contrib",
+    "minimum-stability": "dev",
+    "repositories": [
+        {
+            "type": "composer",
+            "url": "https://packages.drupal.org/8"
+        }
+    ],
+    "require-dev": {
+        "drupal/devel": "^5.0",
+        "drush/drush": "^11 || ^12 || ^13"
+    },
+    "config": {
+        "allow-plugins": true
+    },
+    "extra": {
+        "drupal-scaffold": {
+            "locations": {
+                "web-root": "web"
+            }
+        },
+        "installer-paths": {
+            "web/core": [
+                "type:drupal-core"
+            ],
+            "web/libraries/{$name}": [
+                "type:drupal-library"
+            ],
+            "web/modules/contrib/{$name}": [
+                "type:drupal-module"
+            ],
+            "web/profiles/contrib/{$name}": [
+                "type:drupal-profile"
+            ],
+            "web/themes/contrib/{$name}": [
+                "type:drupal-theme"
+            ],
+            "drush/contrib/{$name}": [
+                "type:drupal-drush"
+            ]
+        }
+    }
+}

tests/testdata/test_drupal_contrib/test_drupal_contrib.info.yml

@@ -0,0 +1,3 @@
+name: 'ddev-drupal-contrib dummy module for tests'
+type: module
+core_version_requirement: ^8.8 || ^9 || ^10 || ^11