[COST-5227] - Add auto large customer logic and XL polling #9265
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| merge_group: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| sanity: | |
| name: Sanity | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Install Python | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: '3.11' | |
| - name: Run pre-commit checks | |
| uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 | |
| env: | |
| SETUPTOOLS_USE_DISTUTILS: stdlib | |
| - name: Check clowdapp manifest | |
| run: bash .github/scripts/check_clowdapp.sh | |
| smokes-labeler: | |
| name: Smoke Test Label | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| fetch-depth: 0 | |
| - name: get Docker image files | |
| id: docker-files | |
| run: .github/scripts/files_require_smokes.sh >> docker-files.txt | |
| - name: add other required files | |
| id: add-files | |
| run: | | |
| echo .dockerignore >> docker-files.txt; | |
| echo Dockerfile >> docker-files.txt; | |
| echo pr_check.sh >> docker-files.txt; | |
| echo deploy/clowdapp.yaml >> docker-files.txt; | |
| - name: Show Dockerfiles | |
| run: cat docker-files.txt | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2 | |
| with: | |
| files_from_source_file: docker-files.txt | |
| - name: Set whether to run tests | |
| id: check-files | |
| run: | | |
| if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then | |
| echo "RUN_TESTS=true" >> $GITHUB_ENV | |
| fi | |
| - name: Set smokes-required label | |
| if: env.RUN_TESTS == 'true' | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| continue-on-error: true | |
| with: | |
| script: | | |
| github.rest.issues.addLabels({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels: [ 'smokes-required' ] | |
| }) | |
| const response = await github.rest.issues.listLabelsOnIssue({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| }) | |
| const labels = response.data.map(({name}) => name) | |
| if (labels.includes('ok-to-skip-smokes')) { | |
| github.rest.issues.removeLabel({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'ok-to-skip-smokes' | |
| }) | |
| } | |
| - name: Set ok-to-skip-smokes label | |
| if: env.RUN_TESTS != 'true' | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| continue-on-error: true | |
| with: | |
| script: | | |
| github.rest.issues.addLabels({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels: [ 'ok-to-skip-smokes' ] | |
| }) | |
| const response = await github.rest.issues.listLabelsOnIssue({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| }) | |
| const labels = response.data.map(({name}) => name) | |
| if (labels.includes('smokes-required')) { | |
| github.rest.issues.removeLabel({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'smokes-required' | |
| }) | |
| } | |
| changed-files: | |
| name: Detect changed files | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2 | |
| with: | |
| files: | | |
| db_functions/** | |
| koku/** | |
| .github/postgres/** | |
| .github/scripts/check_migrations.sh | |
| .github/workflows/ci.yml | |
| Pipfile.lock | |
| - name: Check files or fork | |
| id: check-files-or-fork | |
| run: | | |
| if [ "${{ steps.changed-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then | |
| echo "run_tests=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Show changed files | |
| run: | | |
| echo "Changed files:" | |
| for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do | |
| echo " $file" | |
| done | |
| units: | |
| name: Units - ${{ matrix.python-version }} | |
| needs: changed-files | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python-version: | |
| - '3.11' | |
| env: | |
| COMPOSE_FILE: .github/postgres/docker-compose.yaml | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Display build environment | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: printenv | |
| - name: Start the postgres DB | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: docker compose up -d db | |
| - name: Set up Python ${{ matrix.python-version }} | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pipenv | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: sudo python3 -m pip install pipenv | |
| - name: Cache dependencies | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: cache-dependencies | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: | | |
| ~/.cache/pipenv | |
| ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }} | |
| - name: Install dependencies | |
| if: needs.changed-files.outputs.run_tests == 'true' && steps.cache-dependencies.outputs.cache-hit != 'true' | |
| run: | | |
| pipenv install --dev --deploy --python ${{ matrix.python-version }} | |
| - name: Check migrations | |
| if: | | |
| needs.changed-files.outputs.run_tests == 'true' | |
| && !(contains(github.event.pull_request.labels.*.name, 'skip-migration-check')) | |
| run: bash .github/scripts/check_migrations.sh | |
| env: | |
| DATABASE_SERVICE_NAME: POSTGRES_SQL | |
| DATABASE_ENGINE: postgresql | |
| DATABASE_NAME: postgres | |
| DATABASE_USER: postgres | |
| DATABASE_PASSWORD: postgres | |
| POSTGRES_SQL_SERVICE_HOST: localhost | |
| POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} | |
| PROMETHEUS_MULTIPROC_DIR: /tmp | |
| - name: Run unit tests | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: unit_tests_run | |
| run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/ | |
| env: | |
| DATABASE_SERVICE_NAME: POSTGRES_SQL | |
| DATABASE_ENGINE: postgresql | |
| DATABASE_NAME: postgres | |
| DATABASE_USER: postgres | |
| DATABASE_PASSWORD: postgres | |
| POSTGRES_SQL_SERVICE_HOST: localhost | |
| POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} | |
| HIVE_DATABASE_PASSWORD: hivedbpw | |
| ACCOUNT_ENHANCED_METRICS: True | |
| PROMETHEUS_MULTIPROC_DIR: /tmp | |
| TRINO_DATE_STEP: 31 | |
| MIDDLEWARE_TIME_TO_LIVE: 0 | |
| ENHANCED_ORG_ADMIN: True | |
| ENABLE_S3_ARCHIVING: True | |
| - name: Show test status | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: show_test_status | |
| run: echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'" | |
| - name: Convert coverage report to XML | |
| if: steps.unit_tests_run.outcome == 'success' | |
| run: pipenv run coverage xml | |
| - name: Upload test coverage file | |
| if: steps.unit_tests_run.outcome == 'success' | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 | |
| with: | |
| name: coverage | |
| path: coverage.xml | |
| overwrite: true | |
| coverage: | |
| name: Coverage | |
| needs: [ changed-files,units ] | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: write | |
| statuses: write | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| # this checkout is required for the coverage report. If we don't do this, then | |
| # the uploaded report is invalid and codecov doesn't know how to process it. | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download coverage result from units | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: coverage | |
| - name: Upload coverage to Codecov | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4.4.1 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: Python-${{ matrix.python-version}} | |
| fail_ci_if_error: true | |
| plugin: pycoverage # Only run one plugin even though we don't want any to run. | |
| - name: Set Codecov job status on skipped tests | |
| if: needs.changed-files.outputs.run_tests != 'true' | |
| env: | |
| GITHUB_TOKEN: "${{ github.token }}" | |
| run: | | |
| gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/patch" -f state="success" | |
| gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/project" -f state="success" |