Test SFDX release candidate every week (#3558)

This PR extracts the SFDX integration tests into a reusable workflow so that we can test against the SFDX CLI's release candidates. It is intended to provide early warning of breaking changes. Edited to add: - integration test fixes for v4 that were missed in #3844 - `docutils`, which is required by `rst2ansi`
SFDO-Tooling · Nov 16, 2024 · 1f322d9 · 1f322d9
1 parent 5ae0e51
commit 1f322d9
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 99 deletions.
diff --git a/.github/workflows/chores.yml b/.github/workflows/chores.yml
@@ -12,31 +12,29 @@ jobs:
             hub_version: ${{ steps.devhub-api-version.outputs.hub_version }}
             cci_version: ${{ steps.cci-api-version.outputs.cci_version }}
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   ref: main
-            - name: Set up Python
-              uses: actions/setup-python@v4
             - name: Get Dev Hub API Version
               id: devhub-api-version
               env:
                   HUB_URL: ${{ format('{0}/services/data', secrets.SFDO_HUB_URL) }}
               run: |
                   version=$(curl -s $HUB_URL | jq -r '.[-1] | .version')
-                  echo "::set-output name=hub_version::$version"
+                  echo "hub_version=$version" >> $GITHUB_OUTPUT
             - name: Get CURRENT_SF_API_VERSION
               id: cci-api-version
               run: |
                   version=$(yq '.project.package.api_version' cumulusci/cumulusci.yml)
-                  echo "::set-output name=cci_version::$version"
+                  echo "cci_version=$version" >> $GITHUB_OUTPUT
     update_api_versions:
         runs-on: SFDO-Tooling-Ubuntu
         needs: check_api_versions
         if: ${{ needs.check_api_versions.outputs.hub_version != needs.check_api_versions.outputs.cci_version }}
         env:
             VERSION: ${{ needs.check_api_versions.outputs.hub_version }}
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   fetch-depth: 0
                   ref: main
@@ -58,3 +56,15 @@ jobs:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
               run: |
                   gh pr create --fill --label 'auto-pr'
+    test_sfdx_release_candidate:
+        uses: ./.github/workflows/release_test_sfdx.yml
+        with:
+            sfdx-release-channel: stable-rc
+        secrets:
+            CUMULUSCI_ORG_packaging: ${{ secrets.CUMULUSCI_ORG_packaging }}
+            CUMULUSCI_SERVICE_github: ${{ secrets.CUMULUSCI_SERVICE_github }}
+            CCITEST_APP_KEY: ${{ secrets.CCITEST_APP_KEY }}
+            SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
+            SFDX_HUB_KEY: ${{ secrets.SFDX_HUB_KEY }}
+            SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
+            SFDX_HUB_USERNAME: ${{ secrets.SFDX_HUB_USERNAME }}
diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml
@@ -1,35 +1,9 @@
 name: Release Test
 
 on:
+    workflow_dispatch:
     pull_request:
         types: [opened, synchronize, reopened] # Default
-    workflow_call:
-        secrets:
-            CUMULUSCI_ORG_packaging:
-                required: true
-            CUMULUSCI_SERVICE_github:
-                required: true
-            CCITEST_APP_KEY:
-                required: true
-            SFDX_CLIENT_ID:
-                required: true
-            SFDX_HUB_KEY:
-                required: true
-            SFDX_HUB_KEY_BASE64:
-                required: true
-            SFDX_HUB_USERNAME:
-                required: true
-
-env:
-    CUMULUSCI_ORG_scratch: '{"config_file": "orgs/dev.json", "scratch": true}'
-    CUMULUSCI_ORG_packaging: ${{ secrets.CUMULUSCI_ORG_packaging }}
-    CUMULUSCI_SERVICE_github: ${{ secrets.CUMULUSCI_SERVICE_github }}
-    GITHUB_APP_ID: 129383
-    GITHUB_APP_KEY: ${{ secrets.CCITEST_APP_KEY }}
-    SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
-    SFDX_HUB_KEY: ${{ secrets.SFDX_HUB_KEY }}
-    SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
-    SFDX_HUB_USERNAME: ${{ secrets.SFDX_HUB_USERNAME }}
 
 jobs:
     test_artifacts:
@@ -67,50 +41,14 @@ jobs:
 
     test_release:
         name: "Test Release Flows"
-        runs-on: SFDO-Tooling-Ubuntu
-        concurrency: release
-        steps:
-            - uses: actions/checkout@v3
-            - name: Set up Python 3.11
-              uses: actions/setup-python@v4
-              with:
-                  python-version: 3.11
-                  cache: pip
-                  cache-dependency-path: "pyproject.toml"
-            - name: Set up uv
-              uses: SFDO-Tooling/setup-uv@main
-              with:
-                  version: "0.5.0"
-                  enable-cache: true
-            - name: Install Python dependencies
-              run: uv sync
-            - name: Install sfdx
-              run: |
-                  mkdir sfdx
-                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sfdx/channels/stable/sfdx-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
-                  echo $(realpath sfdx/bin) >> $GITHUB_PATH
-            - name: Authenticate Dev Hub
-              run: |
-                  sfdx plugins --core
-                  echo $SFDX_HUB_KEY_BASE64 | base64 --decode > sfdx.key
-                  sfdx auth:jwt:grant --clientid $SFDX_CLIENT_ID --jwtkeyfile sfdx.key --username $SFDX_HUB_USERNAME --setdefaultdevhubusername -a hub
-            - name: Check out CumulusCI-Test
-              run: |
-                  git clone https://github.com/SFDO-Tooling/CumulusCI-Test
-            - name: Run ci_feature flow
-              run: |
-                  cd CumulusCI-Test
-                  uv run cci flow run ci_feature --org scratch --delete-org
-            - name: Run ci_beta flow
-              run: |
-                  cd CumulusCI-Test
-                  uv run cci flow run ci_beta --org scratch --delete-org
-            - name: Run ci_master flow
-              run: |
-                  cd CumulusCI-Test
-                  uv run cci flow run ci_master --org scratch --delete-org
-            - name: Run release_beta flow
-              run: |
-                  export SFDX_HUB_KEY="$(echo $SFDX_HUB_KEY_BASE64 | base64 --decode)"
-                  cd CumulusCI-Test
-                  uv run cci flow run release_beta --org packaging
+        uses: ./.github/workflows/release_test_sfdx.yml
+        with:
+            sfdx-release-channel: stable
+        secrets:
+            CUMULUSCI_ORG_packaging: ${{ secrets.CUMULUSCI_ORG_packaging }}
+            CUMULUSCI_SERVICE_github: ${{ secrets.CUMULUSCI_SERVICE_github }}
+            CCITEST_APP_KEY: ${{ secrets.CCITEST_APP_KEY }}
+            SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
+            SFDX_HUB_KEY: ${{ secrets.SFDX_HUB_KEY }}
+            SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
+            SFDX_HUB_USERNAME: ${{ secrets.SFDX_HUB_USERNAME }}
diff --git a/.github/workflows/release_test_sfdx.yml b/.github/workflows/release_test_sfdx.yml
@@ -0,0 +1,88 @@
+name: SFDX Integration Test
+
+on:
+    workflow_call:
+        inputs:
+            sfdx-release-channel:
+                required: false
+                type: string
+                default: stable
+        secrets:
+            CUMULUSCI_ORG_packaging:
+                required: true
+            CUMULUSCI_SERVICE_github:
+                required: true
+            CCITEST_APP_KEY:
+                required: true
+            SFDX_CLIENT_ID:
+                required: true
+            SFDX_HUB_KEY:
+                required: true
+            SFDX_HUB_KEY_BASE64:
+                required: true
+            SFDX_HUB_USERNAME:
+                required: true
+
+env:
+    CUMULUSCI_ORG_scratch: '{"config_file": "orgs/dev.json", "scratch": true}'
+    CUMULUSCI_ORG_packaging: ${{ secrets.CUMULUSCI_ORG_packaging }}
+    CUMULUSCI_SERVICE_github: ${{ secrets.CUMULUSCI_SERVICE_github }}
+    GITHUB_APP_ID: 129383
+    GITHUB_APP_KEY: ${{ secrets.CCITEST_APP_KEY }}
+    SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
+    SFDX_HUB_KEY: ${{ secrets.SFDX_HUB_KEY }}
+    SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
+    SFDX_HUB_USERNAME: ${{ secrets.SFDX_HUB_USERNAME }}
+
+jobs:
+    test_release:
+        name: "Test SFDX CLI"
+        runs-on: SFDO-Tooling-Ubuntu
+        concurrency: release
+        steps:
+            - uses: actions/checkout@v4
+            - name: Set up Python 3.11
+              uses: actions/setup-python@v5
+              with:
+                  python-version: 3.11
+                  cache: pip
+                  cache-dependency-path: "pyproject.toml"
+            - name: Set up uv
+              uses: SFDO-Tooling/setup-uv@main
+              with:
+                  version: "0.5.0"
+                  enable-cache: true
+            - name: Install Python dependencies
+              run: uv sync
+            - name: Install Salesforce CLI
+              env:
+                  CHANNEL: ${{ inputs.sfdx-release-channel }}
+              run: |
+                  mkdir sfdx
+                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sf/channels/$CHANNEL/sf-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
+                  echo $(realpath sfdx/bin) >> $GITHUB_PATH
+            - name: Authenticate Dev Hub
+              run: |
+                  sf plugins --core
+                  echo $SFDX_HUB_KEY_BASE64 | base64 --decode > sfdx.key
+                  sf org login jwt --client-id $SFDX_CLIENT_ID --jwt-key-file sfdx.key --username $SFDX_HUB_USERNAME --set-default-dev-hub --alias hub
+            - name: Check out CumulusCI-Test
+              run: |
+                  git clone https://github.com/SFDO-Tooling/CumulusCI-Test
+            - name: Run ci_feature flow
+              run: |
+                  cd CumulusCI-Test
+                  uv run cci flow run ci_feature --org scratch --delete-org
+            - name: Run ci_beta flow
+              run: |
+                  cd CumulusCI-Test
+                  uv run cci flow run ci_beta --org scratch --delete-org
+            - name: Run ci_master flow
+              run: |
+                  cd CumulusCI-Test
+                  uv run cci flow run ci_master --org scratch --delete-org
+            - name: Run release_beta flow
+              run: |
+                  export SFDX_HUB_KEY="$(echo $SFDX_HUB_KEY_BASE64 | base64 --decode)"
+                  cd CumulusCI-Test
+                  uv run cci flow run release_beta --org packaging
diff --git a/.github/workflows/slow_integration_tests.yml b/.github/workflows/slow_integration_tests.yml
@@ -31,20 +31,23 @@ jobs:
                   python-version: 3.11
                   cache: pip
                   cache-dependency-path: "pyproject.toml"
-            - name: Install Python dependencies
-              run: |
-                  python -m pip install -U pip
-                  pip install .[test]
-            - name: Install sfdx
+            - name: Set up uv
+              uses: SFDO-Tooling/setup-uv@main
+              with:
+                  version: "0.5.0"
+                  enable-cache: true
+            - name: Install dependencies
+              run: uv sync -p 3.11
+            - name: Install Salesforce CLI
               run: |
                   mkdir sfdx
-                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sfdx/channels/stable/sfdx-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
+                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sf/channels/stable/sf-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
                   echo $(realpath sfdx/bin) >> $GITHUB_PATH
             - name: Authenticate Dev Hub
               run: |
-                  sfdx plugins --core
+                  sf plugins --core
                   echo $SFDX_HUB_KEY_BASE64 | base64 --decode > sfdx.key
-                  sfdx auth:jwt:grant --clientid $SFDX_CLIENT_ID --jwtkeyfile sfdx.key --username $SFDX_HUB_USERNAME --setdefaultdevhubusername -a hub
+                  sf org login jwt --client-id $SFDX_CLIENT_ID --jwt-key-file sfdx.key --username $SFDX_HUB_USERNAME --set-default-dev-hub --alias hub
               env:
                   SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
                   SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
@@ -54,7 +57,7 @@ jobs:
             - name: Delete scratch org
               if: always()
               run: |
-                  cci org scratch_delete pytest
+                  uv run cci org scratch_delete pytest
     robot_ui:
         name: "Robot: ${{ matrix.job-name }}"
         runs-on: SFDO-Tooling-Ubuntu
@@ -80,26 +83,26 @@ jobs:
                   cache: pip
                   cache-dependency-path: "pyproject.toml"
             - name: Install Python dependencies
-              run: pip install .[test]
-            - name: Install sfdx
+              run: pip install .
+            - name: Install Salesforce CLI
               run: |
                   mkdir sfdx
-                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sfdx/channels/stable/sf-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
+                  wget -qO- https://developer.salesforce.com/media/salesforce-cli/sf/channels/stable/sf-linux-x64.tar.xz | tar xJ -C sfdx --strip-components 1
                   echo $(realpath sfdx/bin) >> $GITHUB_PATH
             - name: Initialize Browser/Playwright
               run: cci robot install_playwright
             - name: Authenticate Dev Hub
               run: |
-                  sfdx plugins --core
+                  sf plugins --core
                   echo $SFDX_HUB_KEY_BASE64 | base64 --decode > sfdx.key
-                  sf org login jwt --client-id $SFDX_CLIENT_ID --jwt-key-file sfdx.key --username $SFDX_HUB_USERNAME --setdefaultdevhubusername -a hub
+                  sf org login jwt --client-id $SFDX_CLIENT_ID --jwt-key-file sfdx.key --username $SFDX_HUB_USERNAME --set-default-dev-hub --alias hub
               env:
                   SFDX_HUB_KEY_BASE64: ${{ secrets.SFDX_HUB_KEY_BASE64 }}
                   SFDX_CLIENT_ID: ${{ secrets.SFDX_CLIENT_ID }}
                   SFDX_HUB_USERNAME: ${{ secrets.SFDX_HUB_USERNAME }}
             - name: Run robot tests
               run: |
-                  coverage run --append $(which cci) task run robot \
+                  cci task run robot \
                     --org ${{ matrix.org-shape }} \
                     -o suites cumulusci/robotframework/tests/salesforce \
                     -o exclude no-browser \

diff --git a/Makefile b/Makefile
@@ -64,13 +64,13 @@ coverage: ## check code coverage quickly with the default Python
 	$(BROWSER) htmlcov/index.html
 
 vcr: # remake VCR cassettes and run other integration tests
-	cci org scratch qa pytest
-	cci org scratch_delete pytest
+	uv run cci org scratch qa pytest
+	uv run cci org scratch_delete pytest
 	find . -name \Test*.yaml | xargs rm
-	pytest --org qa --run-slow-tests -rs --replace-vcrs
+	uv run pytest --org qa --run-slow-tests -rs --replace-vcrs
 
 slow_tests: vcr # remake VCR cassettes and run other integration tests
-	cci org scratch_delete pytest
+	uv run cci org scratch_delete pytest
 	pytest integration_tests/ --org pytest -rs
 
 docs: ## generate Sphinx HTML documentation

diff --git a/pyproject.toml b/pyproject.toml
@@ -54,6 +54,7 @@ dependencies = [
     "simple-salesforce==1.11.4",
     "snowfakery>=4.0.0",
     "xmltodict",
+    "docutils>=0.21.2",
 ]
 
 [dependency-groups]

diff --git a/uv.lock b/uv.lock