CI #70
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI | |
on: | |
push: | |
branches: | |
- trunk | |
pull_request: | |
branches: | |
- trunk | |
schedule: | |
- cron: '0 1 * * *' | |
permissions: | |
actions: write # Needed for skip-duplicate-jobs job | |
contents: read | |
jobs: | |
# Special job which skips duplicate jobs | |
pre_job: | |
name: Skip Duplicate Jobs Pre Job | |
runs-on: ubuntu-latest | |
# Map a step output to a job output | |
outputs: | |
should_skip: ${{ steps.skip_check.outputs.should_skip }} | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@master | |
with: | |
persist-credentials: false | |
submodules: recursive | |
- id: skip_check | |
# NOTE: We store action as submodule since ASF doesn't allow directly referencing external | |
# actions | |
uses: ./.github/actions/skip-duplicate-actions # v4.0.0 | |
with: | |
cancel_others: 'true' | |
github_token: ${{ github.token }} | |
unit_tests: | |
name: Unit Tests (Python ${{ matrix.python_version }}) | |
runs-on: ${{ matrix.os }} | |
timeout-minutes: 8 | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
fail-fast: false | |
matrix: | |
python_version: | |
- 3.7 | |
- 3.8 | |
- 3.9 | |
- "3.10" | |
- "3.11" | |
- "pypy-3.7" | |
os: | |
- ubuntu-latest | |
include: | |
- python_version: pyjion | |
os: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Use Python ${{ matrix.python_version }} | |
if: ${{ matrix.python_version != 'pyjion' }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev | |
- name: Use Python 3.10 | |
if: ${{ matrix.python_version == 'pyjion' }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: "3.10" | |
# From https://github.com/tonybaloney/Pyjion/blob/develop/main/.github/workflows/benchmark.yml#L26 (MIT) | |
- name: Install OS / deb dependencies | |
if: ${{ matrix.python_version == 'pyjion' }} | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq cmake llvm-9 clang-9 autoconf automake \ | |
libtool build-essential python curl git lldb-6.0 liblldb-6.0-dev \ | |
libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev \ | |
libssl-dev libnuma-dev libkrb5-dev zlib1g-dev | |
- name: Setup Dotnet 6 | |
if: ${{ matrix.python_version == 'pyjion' }} | |
uses: actions/[email protected] | |
with: | |
dotnet-version: "6.0.100" | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt', '') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Pyjion | |
if: ${{ matrix.python_version == 'pyjion' }} | |
run: | | |
pip install pyjion | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Run unit tests tox target | |
run: | | |
script -e -c "tox -e py${{ matrix.python_version }}" | |
- name: Run dist install checks tox target | |
if: ${{ matrix.python_version != 'pypy-3.7' && matrix.python_version != 'pyjion' }} | |
run: | | |
script -e -c "tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel" | |
code_coverage: | |
name: Generate Code Coverage | |
runs-on: ubuntu-latest | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Use Python ${{ matrix.python_version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Run Checks | |
run: | | |
script -e -c "tox -e coverage-ci" | |
lint_checks: | |
name: Run Various Lint and Other Checks | |
runs-on: ubuntu-latest | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Use Python ${{ matrix.python_version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-lint.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Run Checks | |
run: | | |
script -e -c "tox -e black-check,isort-check,pyupgrade,checks,import-timings,lint,pylint" | |
build_test_docker_image: | |
name: Build and Verify Docker Image | |
runs-on: ubuntu-latest | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Build Testing Docker Image | |
run: | | |
docker build -f contrib/Dockerfile -t libcloud_runtest_img . | |
- name: Verify Image Works | |
# This step runs checks under various Python versions and it's slow so | |
# we only run it on nightly basis | |
if: ${{ github.event.schedule == '0 1 * * *' }} | |
run: | | |
docker run libcloud_runtest_img | |
security_checks: | |
name: Run Security Checks | |
runs-on: ubuntu-20.04 | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev | |
- name: Use Python ${{ matrix.python_version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-lint.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Install Library Into Virtualenv | |
run: | | |
python -m venv venv/ | |
source venv/bin/activate | |
python -m pip install . | |
- name: Run Pip Audit Check On Main Library Dependencies | |
uses: ./.github/actions/gh-action-pip-audit/ # v1.0.0 | |
with: | |
virtual-environment: venv/ | |
# setuptools which we don't install or depend on directly | |
ignore-vulns: | | |
GHSA-r9hx-vwmv-q579 | |
- name: Cleanup | |
run: | | |
rm -rf venv/ || true | |
- name: Run Pip Audit Check On All Development And Test Dependencies | |
uses: ./.github/actions/gh-action-pip-audit/ # v1.0.0 | |
with: | |
inputs: requirements-tests.txt requirements-lint.txt requirements-mypy.txt requirements-docs.txt | |
# setuptools which we don't install or depend on directly | |
ignore-vulns: | | |
GHSA-r9hx-vwmv-q579 | |
- name: Run Bandit Check | |
run: | | |
script -e -c "tox -e bandit" | |
micro-benchmarks: | |
name: Micro Benchmarks | |
runs-on: ubuntu-latest | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Use Python ${{ matrix.python_version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Run Micro Benchmarks | |
run: | | |
script -e -c "tox -e micro-benchmarks" | |
docs: | |
name: Build and upload Documentation | |
runs-on: ubuntu-latest | |
needs: pre_job | |
if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }} | |
strategy: | |
matrix: | |
python_version: [3.8] | |
steps: | |
- name: Print Environment Info | |
id: printenv | |
run: | | |
printenv | sort | |
- uses: actions/checkout@master | |
with: | |
fetch-depth: 1 | |
- name: Use Python ${{ matrix.python_version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python_version }} | |
- name: Install OS / deb dependencies | |
run: | | |
sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev | |
- name: Cache Python Dependencies | |
uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-docs.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install Python Dependencies | |
run: | | |
pip install "tox==4.4.2" | |
- name: Build Docs | |
run: | | |
script -e -c "tox -e docs-ci" | |
- name: Trigger ReadTheDocs build | |
env: | |
RTD_TOKEN: ${{ secrets.RTD_TOKEN }} | |
run: | | |
pip install requests | |
python ./contrib/trigger_rtd_build.py |