autobuild #3389
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: autobuild | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: ["*"] | |
| push: | |
| branches: [main] | |
| release: | |
| types: [published] | |
| schedule: | |
| # 01:30 UTC = 20:30/21:30 America/New_York = 03:30/04:30 Europe/Athens | |
| - cron: "30 1 * * *" | |
| env: | |
| PYTEST_SPLIT_GROUPS: 6 # How many parallel groups to create for the tests. | |
| PYTEST_TEST_DURATIONS_CACHE_KEY: "test_durations" # Test durations cache key prefix | |
| jobs: | |
| format-check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| cache: pip | |
| cache-dependency-path: ".github/workflows/tiledb-cloud-py.yaml" | |
| - name: Install pre-commit | |
| run: pip install pre-commit | |
| - name: Restore pre-commit cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Pre-commit checks | |
| run: pre-commit run -a -v | |
| run-tests: | |
| runs-on: ${{ matrix.os }} | |
| if: ${{ github.event_name != 'release' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-20.04 | |
| # - macos-12 | |
| python-version: ["3.9"] | |
| dependencies: | |
| - pinned | |
| - fresh | |
| include: | |
| - os: ubuntu-20.04 | |
| path: ~/.cache/pip | |
| # - os: macos-12 | |
| # path: ~/Library/Caches/pip | |
| pytest-split-group: [1, 2, 3, 4, 5, 6] # Range the parallel test groups defined by env.PYTEST_SPLIT_GROUPS | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pinned dependencies | |
| if: ${{ matrix.dependencies == 'pinned' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ matrix.path }} | |
| key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-${{ hashFiles(format('./ci/requirements-py{0}.txt', matrix.python-version)) }} | |
| restore-keys: | | |
| ${{ runner.os }}-py${{ matrix.python-version }}-pip- | |
| # This step ensures that every month we have a new Pip cache, so that | |
| # we don't have one cache sitting for months growing ever-staler. | |
| - name: Get month for cache key | |
| id: month | |
| if: ${{ matrix.dependencies == 'fresh' }} | |
| run: | | |
| date +'cache-month=%Y-%m' >> "$GITHUB_ENV" | |
| - name: Cache fresh install dependencies | |
| if: ${{ matrix.dependencies == 'fresh' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ matrix.path }} | |
| key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-fresh-${{ env.cache-month }} | |
| restore-keys: | | |
| ${{ runner.os }}-py${{ matrix.python-version }}-pip-fresh- | |
| ${{ runner.os }}-py${{ matrix.python-version }}-pip- | |
| - name: Install prerequisites | |
| run: | | |
| pip install --upgrade \ | |
| pip \ | |
| wheel \ | |
| pytest \ | |
| pytest-cov \ | |
| pytest-explicit \ | |
| pytest-split \ | |
| setuptools \ | |
| setuptools-scm | |
| - name: Install pinned dependencies | |
| if: ${{ matrix.dependencies == 'pinned' }} | |
| run: | | |
| pip install -r ci/requirements-py${{ matrix.python-version }}.txt | |
| - name: Install TileDB-Cloud-Py | |
| run: | | |
| pip install .[tests] | |
| pip install tiledb-vector-search --no-deps | |
| # Tries to restore the .test_durations file from the cache to use for splitting tests into groups. | |
| - name: "Attempt to restore .test_durations from cache" | |
| id: test-duration-cache-restore | |
| uses: actions/cache/restore@v4 | |
| continue-on-error: true | |
| with: | |
| path: .test_durations | |
| key: ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}-${{ hashFiles('.test_durations') }} | |
| restore-keys: | | |
| ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}- | |
| - name: "Check .test_durations file" | |
| run: | | |
| if [ -f .test_durations ]; then | |
| echo ".test_durations file exists." | |
| file_content=$(cat .test_durations) | |
| echo $file_content | |
| if [ "$file_content" = "null" ]; then | |
| echo "Removing 'null' .test_durations file." | |
| rm .test_durations | |
| fi | |
| else | |
| echo ".test_durations file does not exist." | |
| fi | |
| - name: Run tests for vendored cloudpickle | |
| # Test pinned dependencies on commit / tag and fresh installs on nightlies | |
| if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| run: | | |
| pip install psutil | |
| pip install src/tiledb/cloud/_vendor/cloudpickle/tests/cloudpickle_testpkg | |
| pytest -sv src/tiledb/cloud/_vendor/cloudpickle | |
| - name: Run tests | |
| # Test pinned dependencies on commit / tag and fresh installs on nightlies | |
| if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| run: | | |
| pytest -sv \ | |
| --tb short \ | |
| --color yes \ | |
| --splits ${{ env.PYTEST_SPLIT_GROUPS }} \ | |
| --group ${{ matrix.pytest-split-group }} \ | |
| --store-durations \ | |
| --splitting-algorithm least_duration \ | |
| --clean-durations \ | |
| -r fExX \ | |
| || .github/scripts/retry 2 \ | |
| pytest -sv \ | |
| --last-failed \ | |
| --tb short \ | |
| --color yes \ | |
| -r fExX | |
| env: | |
| TILEDB_REST_TOKEN: ${{ secrets.TILEDB_CLOUD_HELPER_VAR }} | |
| # Uploads the partial durations from one of the run (specifically from the ubuntu run but it would be the same with macos). | |
| # These partial duration files will be combined into a single .test_durations file and used to update the test cache for the next runs | |
| - name: "Upload partial durations" | |
| uses: actions/upload-artifact@v4 | |
| continue-on-error: true | |
| # Ensure that we save the partial test durations from an ubuntu job that the tests actually run. | |
| if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.9' && (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| with: | |
| name: partial-test-durations-${{ matrix.pytest-split-group }} | |
| path: .test_durations | |
| retention-days: 1 | |
| overwrite: true | |
| # Download and combine the partial test durations uploaded by the run-tests job, | |
| # into a single file and save it to cache for future use. | |
| save-test-durations-to-cache: | |
| name: Download, combine and save test_durations to the actions cache | |
| runs-on: ubuntu-latest | |
| needs: run-tests | |
| continue-on-error: true | |
| env: | |
| FolderPrefix: partial-test-durations | |
| steps: | |
| - name: Download partial durations | |
| uses: actions/download-artifact@v4 | |
| with: | |
| # Download partial test duration artifacts in the same folder | |
| path: partial-test-durations | |
| pattern: partial-test-durations-* | |
| - name: "Combine test duration cache files in .test_durations" | |
| run: | | |
| jq -s add $(find partial-test-durations -type f -name .test_durations) > .test_durations | |
| cat .test_durations | |
| - name: "Save combined test durations to cache" | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .test_durations | |
| key: ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}-${{ hashFiles('.test_durations') }} | |
| upload-to-pypi: | |
| # Upload a wheel only if the entire matrix succeeds. | |
| needs: | |
| - format-check | |
| # Run on release, or on commits to the primary branch. | |
| # Artifacts are uploaded to the real PyPI only when a release is published; | |
| # otherwise, they are sent just to the test instance. | |
| if: > | |
| (github.event_name == 'release' && github.event.action == 'published' && ${{ startsWith(github.event.release.tag_name, 'v') }}) || | |
| (github.event_name == 'push') | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| cache: pip | |
| cache-dependency-path: ci/requirements-py3.9.txt | |
| - name: Make fake version for Test PyPI | |
| if: ${{ github.event_name != 'release' }} | |
| run: | | |
| date -u +'SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0.dev%Y%m%d%H%M' >> "$GITHUB_ENV" | |
| - name: Build wheel | |
| run: | | |
| pip install --upgrade pip build | |
| python -m build | |
| - name: Publish to PyPI | |
| if: ${{ github.event_name == 'release' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| - name: Publish to Test PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| user: __token__ | |
| password: ${{ secrets.TEST_PYPI_API_TOKEN }} | |
| # It's OK if this fails due to an already-existing file, | |
| # if the test publish was run more than once for a revision. | |
| skip-existing: true |