From dc0a0afd27ce0af636abc1696c9cab7e417d9c9e Mon Sep 17 00:00:00 2001
From: Felix Fontein <felix@fontein.de>
Date: Sat, 29 Jun 2024 16:55:58 +0200
Subject: [PATCH] Run some tests with the latest development versions of Docker
 SDK for Python, requests, and urllib3.

---
 .azure-pipelines/azure-pipelines.yml              |  8 ++++----
 .../setup_docker_python_deps/tasks/main.yml       | 10 ++++++++++
 .../setup_docker_sdk_for_python/defaults/main.yml |  5 +++++
 .../setup_docker_sdk_for_python/tasks/main.yml    | 15 +++++++++++++--
 tests/utils/shippable/linux-community.sh          |  9 +++++++++
 tests/utils/shippable/linux.sh                    |  9 +++++++++
 tests/utils/shippable/remote.sh                   |  9 ++++++---
 7 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml
index a4a05a95f..48225d242 100644
--- a/.azure-pipelines/azure-pipelines.yml
+++ b/.azure-pipelines/azure-pipelines.yml
@@ -205,8 +205,8 @@ stages:
         parameters:
           testFormat: devel/{0}
           targets:
-            - name: RHEL 9.4 with latest Docker SDK from PyPi
-              test: rhel/9.4-pypi-latest
+            - name: RHEL 9.4 with Docker SDK, urllib3, requests from sources
+              test: rhel/9.4-dev-latest
           groups:
             - 1
             - 2
@@ -222,8 +222,8 @@ stages:
         parameters:
           testFormat: 2.17/{0}
           targets:
-            - name: RHEL 9.3 with latest Docker SDK from PyPi
-              test: rhel/9.3-pypi-latest
+            - name: RHEL 9.3
+              test: rhel/9.3
           groups:
             - 1
             - 2
diff --git a/tests/integration/targets/setup_docker_python_deps/tasks/main.yml b/tests/integration/targets/setup_docker_python_deps/tasks/main.yml
index 67c2651d2..2bf416f1f 100644
--- a/tests/integration/targets/setup_docker_python_deps/tasks/main.yml
+++ b/tests/integration/targets/setup_docker_python_deps/tasks/main.yml
@@ -13,3 +13,13 @@
     name: "{{ docker_pip_api_packages + (docker_pip_api_packages_python2 if ansible_facts.python.version.major == 2 else []) }}"
     extra_args: "-c {{ remote_constraints }}"
     state: present
+  when: not (force_docker_sdk_for_python_dev | default(false))
+
+- name: Install/upgrade Python requirements from source repositories
+  pip:
+    name:
+      - https://github.com/psf/requests/archive/main.tar.gz
+      - https://github.com/urllib3/urllib3/archive/main.tar.gz
+    extra_args: "-c {{ remote_constraints }}"
+    state: latest
+  when: (force_docker_sdk_for_python_dev | default(false))
diff --git a/tests/integration/targets/setup_docker_sdk_for_python/defaults/main.yml b/tests/integration/targets/setup_docker_sdk_for_python/defaults/main.yml
index 29a257f81..839f21555 100644
--- a/tests/integration/targets/setup_docker_sdk_for_python/defaults/main.yml
+++ b/tests/integration/targets/setup_docker_sdk_for_python/defaults/main.yml
@@ -8,3 +8,8 @@ docker_py_version: '0.0'
 docker_pip_extra_packages: []
 docker_pip_package: docker
 docker_pip_package_limit: ''
+
+docker_pip_git_packages:
+  - https://github.com/psf/requests/archive/main.tar.gz
+  - https://github.com/urllib3/urllib3/archive/main.tar.gz
+  - https://github.com/docker/docker-py/archive/main.tar.gz
diff --git a/tests/integration/targets/setup_docker_sdk_for_python/tasks/main.yml b/tests/integration/targets/setup_docker_sdk_for_python/tasks/main.yml
index 91cd5fcba..257329cce 100644
--- a/tests/integration/targets/setup_docker_sdk_for_python/tasks/main.yml
+++ b/tests/integration/targets/setup_docker_sdk_for_python/tasks/main.yml
@@ -31,9 +31,20 @@
 
     - name: Install/upgrade Python requirements
       pip:
-        name: "{{ [docker_pip_package ~ docker_pip_package_limit] + docker_pip_extra_packages }}"
+        name: >-
+          {{
+            docker_pip_git_packages
+            if (force_docker_sdk_for_python_dev | default(false)) else
+            ([docker_pip_package ~ docker_pip_package_limit] + docker_pip_extra_packages)
+          }}
         extra_args: "-c {{ remote_constraints }}"
-        state: "{{ 'latest' if force_docker_sdk_for_python_pypi | default(false) else 'present' }}"
+        state: >-
+          {{
+            'latest' if (
+              (force_docker_sdk_for_python_pypi | default(false)) or
+              (force_docker_sdk_for_python_dev | default(false))
+            ) else 'present'
+          }}
 
     - name: Check docker-py version
       command: "{{ ansible_python.executable }} -c 'import docker; print(docker.__version__)'"
diff --git a/tests/utils/shippable/linux-community.sh b/tests/utils/shippable/linux-community.sh
index 78dc10a7e..fb1d4be9d 100755
--- a/tests/utils/shippable/linux-community.sh
+++ b/tests/utils/shippable/linux-community.sh
@@ -17,6 +17,15 @@ else
     target="azp/"
 fi
 
+if [[ "${python}" =~ -pypi-latest$ ]]; then
+    python="${python/-pypi-latest}"
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${python}" =~ -dev-latest$ ]]; then
+    python="${python/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
+fi
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --docker "quay.io/ansible-community/test-image:${image}" --python "${python}"
diff --git a/tests/utils/shippable/linux.sh b/tests/utils/shippable/linux.sh
index 9a5381f8c..bef940698 100755
--- a/tests/utils/shippable/linux.sh
+++ b/tests/utils/shippable/linux.sh
@@ -16,6 +16,15 @@ else
     target="azp/"
 fi
 
+if [[ "${image}" =~ -pypi-latest$ ]]; then
+    image="${image/-pypi-latest}"
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${image}" =~ -dev-latest$ ]]; then
+    image="${image/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
+fi
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --docker "${image}"
diff --git a/tests/utils/shippable/remote.sh b/tests/utils/shippable/remote.sh
index 157dd74e1..cd33325d5 100755
--- a/tests/utils/shippable/remote.sh
+++ b/tests/utils/shippable/remote.sh
@@ -27,10 +27,13 @@ else
     target="azp/"
 fi
 
-force_python=""
 if [[ "${version}" =~ -pypi-latest$ ]]; then
     version="${version/-pypi-latest}"
-    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/interation_config.yml
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${version}" =~ -dev-latest$ ]]; then
+    version="${version/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
 fi
 
 stage="${S:-prod}"
@@ -42,4 +45,4 @@ fi
 
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
-    --python "${pyver}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" ${force_python}
+    --python "${pyver}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"