WIP: build-release: add support for running sanity tests

.github/workflows/antsibull-build.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         include:
           - name: Ansible 5 with default settings
-            options: '-e antsibull_ansible_version=5.99.0'
+            options: '-e antsibull_ansible_version=5.99.0 -e antsibull_run_sanity=true'
           - name: Ansible 6 with ansible-core devel
             options: '-e antsibull_ansible_version=6.99.0 -e antsibull_build_file=ansible-6.build -e antsibull_data_dir="{{ antsibull_data_git_dir }}/6" -e antsibull_ansible_git_version=devel'
 

playbooks/nested-ansible-tests.yaml

@@ -4,7 +4,7 @@
   hosts: localhost
   gather_facts: no
   vars:
-    # These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests.yaml
+    # These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests/nested.yaml
     antsibull_sdist_dir: "{{ playbook_dir | dirname }}/build"
     antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv"
     antsibull_ansible_git_dir: "{{ antsibull_sdist_dir }}/ansible"

roles/build-release/defaults/main.yaml

@@ -46,6 +46,10 @@ antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv"
 # Whether or not to start from scratch with a new venv if one exists
 antsibull_venv_cleanup: true
 
+# Whether to run "ansible-test sanity --docker" with podman on packaged collections
+# This is disabled by default since it is time consuming to run.
+antsibull_run_sanity: false
+
 # TODO:
 # --dest-data-dir (Directory to write .build and .deps files to, as well as changelog and porting guide if applicable. Defaults to --data-dir)
 # --collection-cache (Directory of cached collection tarballs. Will be used if a collection tarball to be downloaded exists in here, and will be populated when downloading new tarballs.)

roles/build-release/tasks/tests.yaml

@@ -20,108 +20,12 @@
     virtualenv: "{{ antsibull_ansible_venv }}"
     virtualenv_command: "{{ ansible_python.executable }} -m venv"
 
-# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >=
-- block:
-    # Note: the value is either _ansible_base_version or _ansible_core_version depending on the version
-    # ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps
-    - name: Retrieve the expected minimum version of ansible-core
-      shell: >-
-        grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}'
-      changed_when: false
-      register: _expected_ansible_core
+- name: Test and validate that we have the expected versions of collections, ansible-core and ansible
+  include_tasks: tests/versions.yaml
 
-    - name: Retrieve the installed version of ansible-core
-      shell: >-
-        {{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}'
-      changed_when: false
-      register: _installed_ansible_core
+- name: Install the built ansible in venv and run integration tests with it
+  include_tasks: tests/nested.yaml
 
-    - name: Validate the version of ansible-core
-      ansible.builtin.assert:
-        that:
-          - _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=')
-        success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}"
-        fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}"
-
-- block:
-    - name: Retrieve expected versions of Ansible and collections
-      command: cat {{ antsibull_data_dir }}/{{ _deps_file }}
-      changed_when: false
-      register: _expected_versions
-
-    - name: Retrieve collections that should be included in the package
-      shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#"
-      changed_when: false
-      register: _included_collections
-
-    - name: Retrieve the installed version of ansible with pip
-      shell: >-
-        {{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}'
-      changed_when: false
-      register: _ansible_version_pypi
-
-    - name: Retrieve the builtin reported version of ansible
-      command: >-
-        {{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)'
-      changed_when: false
-      register: _ansible_version_builtin
-
-    - name: Validate the version of ansible
-      ansible.builtin.assert:
-        that:
-          - "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout"
-          - _ansible_version_pypi.stdout == _ansible_version_builtin.stdout
-        success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'"
-        fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'"
-
-    - name: Retrieve installed collections
-      environment:
-        # In case we happen to be testing with devel, don't print a warning about it
-        ANSIBLE_DEVEL_WARNING: false
-        # Until https://github.com/ansible/ansible/pull/70173 is backported and released
-        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections"
-      # List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file
-      shell: >-
-        {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }'
-      changed_when: false
-      register: _installed_collections
-
-    - name: Validate that the installed collections are the expected ones
-      ansible.builtin.assert:
-        that:
-          - item in _expected_versions.stdout
-        success_msg: "{{ item }} matches {{ _deps_file }}"
-        fail_msg: "{{ item }} does not match {{ _deps_file }}"
-      loop: "{{ _installed_collections.stdout_lines }}"
-
-    - name: Validate that included collections are packaged
-      ansible.builtin.assert:
-        that:
-          - item in _installed_collections.stdout
-        success_msg: "{{ item }} is in ansible.in and inside the package"
-        fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?"
-      loop: "{{ _included_collections.stdout_lines }}"
-
-- block:
-    - name: Create a temporary COLLECTIONS_PATH
-      file:
-        path: "{{ antsibull_sdist_dir }}/ansible_collections"
-        state: directory
-
-    - name: Install community.general for tests using 'a_module' and 'collection_version'
-      environment:
-        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
-      command: >-
-        {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general
-
-    - name: Run nested Ansible tests with the Ansible we just built
-      environment:
-        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
-      command: >-
-        {{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local
-          -e antsibull_sdist_dir="{{ antsibull_sdist_dir }}"
-          -e antsibull_ansible_venv="{{ antsibull_ansible_venv }}"
-          -e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}"
-          -e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}"
-          -e _python_version="{{ _python_version }}"
-          {{ playbook_dir }}/nested-ansible-tests.yaml
+- name: Run ansible-test sanity --docker on installed collections with podman
+  include_tasks: tests/sanity.yaml
+  when: antsibull_run_sanity | bool

roles/build-release/tasks/tests/nested.yaml

@@ -0,0 +1,23 @@
+- block:
+    - name: Create a temporary COLLECTIONS_PATH
+      file:
+        path: "{{ antsibull_sdist_dir }}/ansible_collections"
+        state: directory
+
+    - name: Install community.general for tests using 'a_module' and 'collection_version'
+      environment:
+        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
+      command: >-
+        {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general
+
+    - name: Run nested Ansible tests with the Ansible we just built
+      environment:
+        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections"
+      command: >-
+        {{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local
+          -e antsibull_sdist_dir="{{ antsibull_sdist_dir }}"
+          -e antsibull_ansible_venv="{{ antsibull_ansible_venv }}"
+          -e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}"
+          -e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}"
+          -e _python_version="{{ _python_version }}"
+          {{ playbook_dir }}/nested-ansible-tests.yaml

roles/build-release/tasks/tests/sanity.yaml

@@ -0,0 +1,55 @@
+- name: Get list of installed packages to verify if podman is installed
+  package_facts:
+    manager: "auto"
+  # This is noisy (and potentially sensitive) to print on stdout
+  no_log: true
+
+# This is so we can otherwise run unprivileged if podman is already installed
+- when: "'podman' not in ansible_facts['packages'].keys()"
+  become: true
+  block:
+    - name: Install podman
+      package:
+        name: podman
+        state: present
+  rescue:
+    - name: Could not install podman
+      fail:
+        msg: |
+          Failed to elevate privileges and install podman.
+          Install podman manually or run ansible-playbook with elevated privileges.
+
+# The tests are no longer installed on ansible>=6.0, use the ones from the release tarball
+- name: Unzip the release tarball
+  ansible.builtin.unarchive:
+    src: "{{ _release_archive }}"
+    dest: "{{ antsibull_sdist_dir }}"
+    remote_src: yes
+
+# ansible.builtin.find doesn't have mindepth
+# https://github.com/ansible/ansible/issues/36369
+- name: Find collection directories
+  vars:
+    _extracted_release: "{{ antsibull_sdist_dir }}/ansible-{{ antsibull_ansible_version }}"
+  command: find {{ _extracted_release }}/ansible_collections -mindepth 2 -maxdepth 2 -type d
+  register: _collection_directories
+
+# Note: unless we run a git init first, ansible-test won't be able to pick up tests because it's under
+# a .gitignore directory (antsibull/build)
+- name: Run sanity tests and collect results at the end
+  shell: "git init . -b main; {{ antsibull_ansible_venv }}/bin/ansible-test sanity --skip-test pylint --docker"
+  ignore_errors: yes
+  args:
+    chdir: "{{ item }}"
+  loop: "{{ _collection_directories.stdout_lines }}"
+  register: _sanity_tests
+
+- name: Print sanity test failures
+  debug:
+    msg: |
+      {{ _result.item }}
+      {{ _result.stderr }}
+  when: _result.rc == 1
+  loop: "{{ _sanity_tests.results }}"
+  loop_control:
+    loop_var: _result

roles/build-release/tasks/tests/versions.yaml

@@ -0,0 +1,84 @@
+---
+# Validate that we have the expected versions for the Ansible collections as well ansible-core and ansible
+
+# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >=
+- block:
+    # Note: the value is either _ansible_base_version or _ansible_core_version depending on the version
+    # ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps
+    - name: Retrieve the expected minimum version of ansible-core
+      shell: >-
+        grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}'
+      changed_when: false
+      register: _expected_ansible_core
+
+    - name: Retrieve the installed version of ansible-core
+      shell: >-
+        {{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}'
+      changed_when: false
+      register: _installed_ansible_core
+
+    - name: Validate the version of ansible-core
+      ansible.builtin.assert:
+        that:
+          - _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=')
+        success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}"
+        fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}"
+
+- block:
+    - name: Retrieve expected versions of Ansible and collections
+      command: cat {{ antsibull_data_dir }}/{{ _deps_file }}
+      changed_when: false
+      register: _expected_versions
+
+    - name: Retrieve collections that should be included in the package
+      shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#"
+      changed_when: false
+      register: _included_collections
+
+    - name: Retrieve the installed version of ansible with pip
+      shell: >-
+        {{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}'
+      changed_when: false
+      register: _ansible_version_pypi
+
+    - name: Retrieve the builtin reported version of ansible
+      command: >-
+        {{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)'
+      changed_when: false
+      register: _ansible_version_builtin
+
+    - name: Validate the version of ansible
+      ansible.builtin.assert:
+        that:
+          - "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout"
+          - _ansible_version_pypi.stdout == _ansible_version_builtin.stdout
+        success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'"
+        fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'"
+
+    - name: Retrieve installed collections
+      environment:
+        # In case we happen to be testing with devel, don't print a warning about it
+        ANSIBLE_DEVEL_WARNING: false
+        # Until https://github.com/ansible/ansible/pull/70173 is backported and released
+        ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections"
+      # List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file
+      shell: >-
+        {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }'
+      changed_when: false
+      register: _installed_collections
+
+    - name: Validate that the installed collections are the expected ones
+      ansible.builtin.assert:
+        that:
+          - item in _expected_versions.stdout
+        success_msg: "{{ item }} matches {{ _deps_file }}"
+        fail_msg: "{{ item }} does not match {{ _deps_file }}"
+      loop: "{{ _installed_collections.stdout_lines }}"
+
+    - name: Validate that included collections are packaged
+      ansible.builtin.assert:
+        that:
+          - item in _installed_collections.stdout
+        success_msg: "{{ item }} is in ansible.in and inside the package"
+        fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?"
+      loop: "{{ _included_collections.stdout_lines }}"