diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml deleted file mode 100644 index a5112553..00000000 --- a/.github/workflows/github-ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: QMI Python CI runner on push - -# This runner runs on regular push to a branch. -on: - push: - paths-ignore: - - README.md - - CHANGELOG.md - - .gitignore - - ACKNOWLEDGEMENTS.md - - LICENSE.md - - CONTRIBUTING.md - - TESTING.md - -env: - # minimum pylint code quality score (max. 10) - PYLINT_MIN_SCORE: "9.00" - # minimum code coverage (goal: 90%; max. 100%) - COVERAGE_MIN_PERC: "90" - # maximum code complexity (goal: <= 30; unbounded) - COMPLEXITY_MAX_SCORE: "30" - # this should be a comma separated list - SOURCE_DIRS: "qmi/" - -jobs: - on_push: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python-latest - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: | - sudo apt-get update -qy - sudo apt-get install -y bc - python3 --version - pip install --upgrade pip - pip install -e '.[dev]' - - name: pylint - if: always() - run: | - pylint --score=yes --load-plugins=pylint.extensions.mccabe --max-complexity=$COMPLEXITY_MAX_SCORE $SOURCE_DIRS | tee pylint.log - PYLINT_SCORE=$( tail -n 2 pylint.log | grep -o '[0-9]\{1,2\}\.[0-9]\{2\}' | head -n 1 ) - echo "Pylint score: $PYLINT_SCORE" - exit $( echo "$PYLINT_SCORE < $PYLINT_MIN_SCORE" | bc ) - - name: upload pylint test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: pylint-results - path: pylint.log - - name: mypy - if: always() - run: | - mypy --namespace-packages $SOURCE_DIRS | tee mypy.log - if [ -n "$( tail -n 1 mypy.log | grep -e '^Succes' )" ]; then RESULT="pass"; else RESULT="fail"; fi - echo "Mypy result: $RESULT" - if [[ "$RESULT" = *"pass"* ]]; then exit 0; else exit 1; fi - - name: upload mypy test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: mypy-results - path: mypy.log - - name: coverage - if: always() - run: | - coverage run --branch --source=$SOURCE_DIRS -m unittest discover --start-directory=tests --pattern="test_*.py" - coverage report --show-missing --fail-under=$COVERAGE_MIN_PERC | tee coverage.log - COVERAGE_PERC=$(grep "TOTAL" coverage.log | grep -Eo '[0-9.]+%' | sed 's/%//') - echo "Coverage: $COVERAGE_PERC%" - exit $( echo "$COVERAGE_PERC < $COVERAGE_MIN_PERC" | bc ) - - name: upload coverage results - if: always() - uses: actions/upload-artifact@v3 - with: - name: coverage-results - path: coverage.log - diff --git a/.github/workflows/pull-request-ci.yml b/.github/workflows/pull-request-ci.yml new file mode 100644 index 00000000..f564a326 --- /dev/null +++ b/.github/workflows/pull-request-ci.yml @@ -0,0 +1,51 @@ +name: QMI Python CI runner on pull request + +on: + pull_request: + branches: [ "main" ] + paths-ignore: + - README.md + - CHANGELOG.md + - .gitignore + - ACKNOWLEDGEMENTS.md + - LICENSE.md + - CONTRIBUTING.md + - TESTING.md + +env: + BADGES_DIR: ".github/badges" + +jobs: + build: + strategy: + max-parallel: 3 + matrix: + python-version: ["3.8", "3.9", "3.10"] + + uses: ./.github/workflows/reusable-ci-workflows.yml + with: + python-version: ${{ matrix.python-version }} + ref: ${{ github.head_ref }} + + create-badges: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Generate and push badges + if: ${{ matrix.python-version == '3.10' }} + run: | + pip install anybadge + anybadge -o -l pylint -v $(tail -n 2 pylint.log | grep -o '[0-9]\{1,2\}\.[0-9]\{2\}' | head -n 1) -f $BADGES_DIR/pylint.svg 2=red 4=orange 8=yellow 10=green + anybadge -o -l mypy -v $([ -n "$(tail -n 1 mypy.log | grep -e '^Succes')" ] && echo pass || echo fail) -f $BADGES_DIR/mypy.svg fail=red pass=green + COVERAGE_PERC=$(grep "TOTAL" coverage.log | grep -Eo '[0-9.]+%' | sed 's/%//') + anybadge -o -l coverage -v $COVERAGE_PERC -f $BADGES_DIR/coverage.svg 60=red 80=orange 100=green + git config user.name "Badge Bot" + git config user.email "<>" + git add $BADGES_DIR/*.svg + git commit -m "Update badges" || true + git push || true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index 64ede837..00000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: QMI Python CI runner on pull request - -# This runner runs on a pull request to main. -on: - pull_request: - branches: [ "main" ] - paths-ignore: - - README.md - - CHANGELOG.md - - .gitignore - - ACKNOWLEDGEMENTS.md - - LICENSE.md - - CONTRIBUTING.md - - TESTING.md - -env: - # minimum pylint code quality score (max. 10) - PYLINT_MIN_SCORE: "9.00" - # minimum code coverage (goal: 90%; max. 100%) - COVERAGE_MIN_PERC: "90" - # maximum code complexity (goal: <= 30; unbounded) - COMPLEXITY_MAX_SCORE: "30" - # this should be a comma separated list - SOURCE_DIRS: "qmi/" - BADGES_DIR: ".github/badges" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 3 - matrix: - python-version: ["3.8", "3.9", "3.10"] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update -qy - sudo apt-get install -y bc - pip install --upgrade pip - pip install -e '.[dev]' - pip install anybadge - - name: setup git config - run: | - git config user.name "Badge Bot" - git config user.mail "<>" - chmod u+w $BADGES_DIR - - name: pylint - if: always() - run: | - pylint --score=yes --load-plugins=pylint.extensions.mccabe --max-complexity=$COMPLEXITY_MAX_SCORE $SOURCE_DIRS | tee pylint.log - SCORE=$( tail -n 2 pylint.log | grep -o '[0-9]\{1,2\}\.[0-9]\{2\}' | head -n 1 ) - echo "Pylint score: $SCORE" - if [ "${{ matrix.python-version }}" = "3.10" ]; then anybadge -o -l pylint -v $SCORE -f $BADGES_DIR/pylint.svg 2=red 4=orange 8=yellow 10=green; fi - exit $( echo "$SCORE < $PYLINT_MIN_SCORE" | bc ) - - name: upload pylint test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: pylint-results - path: pylint.log - - name: mypy - if: always() - run: | - mypy --namespace-packages $SOURCE_DIRS | tee mypy.log - if [ -n "$( tail -n 1 mypy.log | grep -e '^Succes' )" ]; then RESULT="pass"; else RESULT="fail"; fi - echo "Mypy result: $RESULT" - if [ "${{ matrix.python-version }}" = "3.10" ]; then anybadge -o -l mypy -v $RESULT -f $BADGES_DIR/mypy.svg fail=red pass=green; fi - if [[ "$RESULT" = *"pass"* ]]; then exit 0; else exit 1; fi - - name: upload mypy test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: mypy-results - path: mypy.log - - name: coverage - if: always() - run: | - if coverage run --branch --source=$SOURCE_DIRS -m unittest discover --start-directory=tests --pattern="test_*.py"; then RESULT="pass"; else RESULT="fail"; fi - coverage report --show-missing --fail-under=$COVERAGE_MIN_PERC | tee coverage.log - COVERAGE_PERC=$(grep "TOTAL" coverage.log | grep -Eo '[0-9.]+%' | sed 's/%//') - echo "Coverage: $COVERAGE_PERC%" - exit $( echo "$COVERAGE_PERC < $COVERAGE_MIN_PERC" | bc ) - if [ "${{ matrix.python-version }}" = "3.10" ]; then anybadge -o -l coverage -v $COVERAGE_PERC -f $BADGES_DIR/coverage.svg 60=red 80=orange 100=green; fi - if [ "${{ matrix.python-version }}" = "3.10" ]; then anybadge -o -l tests -v $RESULT -f $BADGES_DIR/tests.svg fail=red pass=green; fi - - name: upload coverage results - if: always() - uses: actions/upload-artifact@v3 - with: - name: coverage-results - path: coverage.log - - name: push all internal commits - if: always() - run: | - if [ "${{ matrix.python-version }}" = "3.10" ]; then git add $BADGES_DIR/pylint.svg; fi - if [ "${{ matrix.python-version }}" = "3.10" ]; then git add $BADGES_DIR/mypy.svg; fi - if [ "${{ matrix.python-version }}" = "3.10" ]; then git add $BADGES_DIR/coverage.svg; fi - if [ "${{ matrix.python-version }}" = "3.10" ]; then git add $BADGES_DIR/tests.svg; fi - if [ "${{ matrix.python-version }}" = "3.10" ]; then if git commit -m "commit badges" 2>/dev/null; then git push; fi; fi diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml new file mode 100644 index 00000000..f30c5c3c --- /dev/null +++ b/.github/workflows/push-ci.yml @@ -0,0 +1,18 @@ +name: QMI Python CI runner on push + +on: + push: + paths-ignore: + - README.md + - CHANGELOG.md + - .gitignore + - ACKNOWLEDGEMENTS.md + - LICENSE.md + - CONTRIBUTING.md + - TESTING.md + +jobs: + on_push: + uses: ./.github/workflows/reusable-ci-workflows.yml + with: + python-version: "3.11" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/pypi-publish.yml similarity index 95% rename from .github/workflows/python-publish.yml rename to .github/workflows/pypi-publish.yml index 351a4187..49fd82ce 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -27,9 +27,9 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/reusable-ci-workflows.yml b/.github/workflows/reusable-ci-workflows.yml new file mode 100644 index 00000000..f025a93d --- /dev/null +++ b/.github/workflows/reusable-ci-workflows.yml @@ -0,0 +1,77 @@ +name: Python CI Reusable Workflow + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + ref: + required: false + type: string + +env: + PYLINT_MIN_SCORE: "9.00" + COVERAGE_MIN_PERC: "90" + COMPLEXITY_MAX_SCORE: "30" + SOURCE_DIRS: "qmi/" + BADGES_DIR: ".github/badges" + +jobs: + ci-checks: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install dependencies + run: | + sudo apt-get update -qy + sudo apt-get install -y bc + pip install --upgrade pip + pip install -e '.[dev]' + + - name: Run pylint + run: | + pylint --score=yes --load-plugins=pylint.extensions.mccabe --max-complexity=$COMPLEXITY_MAX_SCORE $SOURCE_DIRS | tee pylint-${{ inputs.python-version }}.log + PYLINT_SCORE=$(tail -n 2 pylint-${{ inputs.python-version }}.log | grep -o '[0-9]\{1,2\}\.[0-9]\{2\}' | head -n 1) + echo "Pylint score: $PYLINT_SCORE" + exit $(echo "$PYLINT_SCORE < $PYLINT_MIN_SCORE" | bc) + - name: Upload pylint results + uses: actions/upload-artifact@v4 + with: + name: pylint-results-${{ inputs.python-version }} + path: pylint-${{ inputs.python-version }}.log + + - name: Run mypy + run: | + mypy --namespace-packages $SOURCE_DIRS | tee mypy-${{ inputs.python-version }}.log + if [ -n "$(tail -n 1 mypy-${{ inputs.python-version }}.log | grep -e '^Succes')" ]; then RESULT="pass"; else RESULT="fail"; fi + echo "Mypy result: $RESULT" + exit $([[ "$RESULT" == "pass" ]] && echo 0 || echo 1) + - name: Upload mypy results + uses: actions/upload-artifact@v4 + with: + name: mypy-results-${{ inputs.python-version }} + path: mypy-${{ inputs.python-version }}.log + + - name: Run coverage + run: | + coverage run --branch --source=$SOURCE_DIRS -m unittest discover --start-directory=tests --pattern="test_*.py" + coverage report --show-missing --fail-under=$COVERAGE_MIN_PERC | tee coverage-${{ inputs.python-version }}.log + COVERAGE_PERC=$(grep "TOTAL" coverage-${{ inputs.python-version }}.log | grep -Eo '[0-9.]+%' | sed 's/%//') + echo "Coverage: $COVERAGE_PERC%" + exit $(echo "$COVERAGE_PERC < $COVERAGE_MIN_PERC" | bc) + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-results-${{ inputs.python-version }} + path: coverage-${{ inputs.python-version }}.log diff --git a/.github/workflows/schedule-full-ci-test-main.yml b/.github/workflows/schedule-full-ci-test-main.yml deleted file mode 100644 index 701c9cbf..00000000 --- a/.github/workflows/schedule-full-ci-test-main.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: QMI Python CI full test runner - -on: - schedule: - # Schedule for every Monday at 7am - - cron: "0 7 * * 1" - -env: - # minimum pylint code quality score (max. 10) - PYLINT_MIN_SCORE: "9.00" - # minimum code coverage (goal: 90%; max. 100%) - COVERAGE_MIN_PERC: "90" - # maximum code complexity (goal: <= 30; unbounded) - COMPLEXITY_MAX_SCORE: "30" - # this should be a comma separated list - SOURCE_DIRS: "qmi/" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python-latest - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update -qy - sudo apt-get install -y bc - python3 --version - pip install --upgrade pip - pip install -e '.[dev]' - pip install anybadge - - name: pylint - if: always() - run: | - pylint --score=yes --load-plugins=pylint.extensions.mccabe --max-complexity=$COMPLEXITY_MAX_SCORE $SOURCE_DIRS | tee pylint.log - SCORE=$( tail -n 2 pylint.log | grep -o '[0-9]\{1,2\}\.[0-9]\{2\}' | head -n 1 ) - echo "Pylint score: $SCORE" - exit $( echo "$SCORE < $PYLINT_MIN_SCORE" | bc ) - - name: upload pylint test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: pylint-results - path: pylint.log - - name: mypy - if: always() - run: | - mypy --namespace-packages $SOURCE_DIRS | tee mypy.log - if [ -n "$( tail -n 1 mypy.log | grep -e '^Succes' )" ]; then RESULT="passed"; else RESULT="failed"; fi - echo "Mypy result: $RESULT" - if [[ "$RESULT" = *"pass"* ]]; then exit 0; else exit 1; fi - - name: upload mypy test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: mypy-results - path: mypy.log - - name: coverage - if: always() - run: | - coverage run --branch --source=$SOURCE_DIRS -m unittest discover --start-directory=tests --pattern="test_*.py" - coverage report --show-missing --fail-under=$COVERAGE_MIN_PERC | tee coverage.log - COVERAGE_PERC=$(grep "TOTAL" coverage.log | grep -Eo '[0-9.]+%' | sed 's/%//') - echo "Coverage: $COVERAGE_PERC%" - exit $( echo "$COVERAGE_PERC < $COVERAGE_MIN_PERC" | bc ) - - name: upload coverage results - if: always() - uses: actions/upload-artifact@v3 - with: - name: coverage-results - path: coverage.log - - name: unit-tests latest - if: always() - run: | - pip install unittest-xml-reporting - python -m xmlrunner --output-file testresults.xml discover --start-directory=tests --pattern="test_*.py" - - name: upload unit-tests results - if: always() - uses: actions/upload-artifact@v3 - with: - name: unit-test-results - path: testresults.xml diff --git a/.github/workflows/scheduled-full-ci.yml b/.github/workflows/scheduled-full-ci.yml new file mode 100644 index 00000000..f9e561c0 --- /dev/null +++ b/.github/workflows/scheduled-full-ci.yml @@ -0,0 +1,40 @@ +name: QMI Python CI full test runner + +on: + schedule: + # Schedule for every Monday at 7am + - cron: "0 7 * * 1" + +jobs: + full-test: + strategy: + max-parallel: 1 + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + uses: ./.github/workflows/reusable-ci-workflows.yml + with: + python-version: ${{ matrix.python-version }} + ref: ${{ github.head-ref }} + + unit-tests: + runs-on: ubuntu-latest + needs: full-test + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Run unit tests and generate report + if: always() + run: | + pip install unittest-xml-reporting + python -m xmlrunner --output-file testresults.xml discover --start-directory=tests --pattern="test_*.py" + + - name: Upload unit test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: unit-test-results-${{ matrix.python-version }} + path: testresults.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 50eb3b84..9fb8f8f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## \[x.y.z] - Unreleased +### Changed +- The CI pipelines are now using reusable workflows, placed in reusable-ci-workflows.yml. +- The file names for the different pipeline actions were also changed to be more descriptive. + ### Fixed - mypy error on `config_struct.py` by adding extra logic check `and not isinstance(val, type)` on L236. - Also made in `config_struct.py` in L186 also tuples to be recognized as "untyped values". +- workflow artifacts to be of @v4 instead of @v3 that are to be deprecated. For `setup-python` @v5 even. ## [0.45.0] - 2024-07-19