Skip to content

Commit

Permalink
Merge branch '3006.x' into test_batch_9
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmcmarrow
cmcmarrow authored Sep 29, 2023
2 parents dd83b91 + 309829e commit 3b2efac
Show file tree
Hide file tree
Showing 133 changed files with 11,658 additions and 4,373 deletions.
24 changes: 17 additions & 7 deletions .github/workflows/build-deb-packages.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
name: Build Debian Packages
name: Build DEB Packages

on:
workflow_call:
Expand All @@ -16,6 +16,17 @@ on:
required: true
type: string
description: The version of python to use with relenv
source:
required: true
type: string
description: The backend to build the packages with

env:
COLUMNS: 190
AWS_MAX_ATTEMPTS: "10"
AWS_RETRY_MODE: "adaptive"
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple

jobs:
build:
Expand All @@ -31,19 +42,18 @@ jobs:
- x86_64
- aarch64
source:
- onedir
- src
- ${{ inputs.source }}

container:
image: ghcr.io/saltstack/salt-ci-containers/packaging:debian-12

steps:
# Checkout here so we can easily use custom actions
- uses: actions/checkout@v3
- uses: actions/checkout@v4

# Checkout here for the build process
- name: Checkout in build directory
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
path:
pkgs/checkout/
Expand Down Expand Up @@ -89,7 +99,7 @@ jobs:
working-directory: pkgs/checkout/
run: |
tools pkg build deb --relenv-version=${{ inputs.relenv-version }} --python-version=${{ inputs.python-version }} ${{
matrix.source == 'onedir' &&
inputs.source == 'onedir' &&
format('--onedir=salt-{0}-onedir-linux-{1}.tar.xz', inputs.salt-version, matrix.arch)
||
format('--arch={0}', matrix.arch)
Expand All @@ -102,7 +112,7 @@ jobs:
- name: Set Artifact Name
id: set-artifact-name
run: |
if [ "${{ matrix.source }}" != "src" ]; then
if [ "${{ inputs.source }}" != "src" ]; then
echo "artifact-name=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-deb" >> "$GITHUB_OUTPUT"
else
echo "artifact-name=salt-${{ inputs.salt-version }}-${{ matrix.arch }}-deb-from-src" >> "$GITHUB_OUTPUT"
Expand Down
134 changes: 134 additions & 0 deletions .github/workflows/build-deps-ci-action-macos.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
---
name: Test Dependencies

on:
workflow_call:
inputs:
distro-slug:
required: true
type: string
description: The OS slug to run tests against
nox-session:
required: true
type: string
description: The nox session to run
salt-version:
type: string
required: true
description: The Salt version to set prior to running tests.
cache-prefix:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
nox-version:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
description: The onedir package name to use
default: salt


env:
COLUMNS: 190
PIP_INDEX_URL: "https://pypi-proxy.saltstack.net/root/local/+simple/"
PIP_EXTRA_INDEX_URL: "https://pypi.org/simple"

jobs:

dependencies:
name: Test Dependencies
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 90
steps:

- name: "Throttle Builds"
shell: bash
run: |
t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"
- name: Checkout Source Code
uses: actions/checkout@v4

- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}

- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/

- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"

- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
brew install openssl@3
- name: Install Nox
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
env:
PRINT_TEST_SELECTION: "0"
PRINT_SYSTEM_INFO: "0"
run: |
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --install-only -e ${{ inputs.nox-session }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e compress-dependencies -- ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*
158 changes: 158 additions & 0 deletions .github/workflows/build-deps-ci-action.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
---
name: Test Dependencies

on:
workflow_call:
inputs:
distro-slug:
required: true
type: string
description: The OS slug to run tests against
nox-session:
required: true
type: string
description: The nox session to run
salt-version:
type: string
required: true
description: The Salt version to set prior to running tests.
cache-prefix:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
nox-version:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
description: The onedir package name to use
default: salt


env:
COLUMNS: 190
AWS_MAX_ATTEMPTS: "10"
AWS_RETRY_MODE: "adaptive"
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple

jobs:

dependencies:
name: Test Dependencies
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:

- name: "Throttle Builds"
shell: bash
run: |
t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
- name: Checkout Source Code
uses: actions/checkout@v4

- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}

- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/

- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: PyPi Proxy
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts

- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*
Loading

0 comments on commit 3b2efac

Please sign in to comment.