tiledb-cloud-py.yaml

name: autobuild

on:
  workflow_dispatch:
  pull_request:
    branches: ["*"]
  push:
    branches: [main]
  release:
    types: [published]
  schedule:
    # 01:30 UTC = 20:30/21:30 America/New_York = 03:30/04:30 Europe/Athens
    - cron: "30 1 * * *"

env:
  PYTEST_SPLIT_GROUPS: 6 # How many parallel groups to create for the tests.
  PYTEST_TEST_DURATIONS_CACHE_KEY: "test_durations" # Test durations cache key prefix

jobs:
  format-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          cache: pip
          cache-dependency-path: ".github/workflows/tiledb-cloud-py.yaml"
      - name: Install pre-commit
        run: pip install pre-commit
      - name: Restore pre-commit cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Pre-commit checks
        run: pre-commit run -a -v

  run-tests:
    runs-on: ${{ matrix.os }}
    if: ${{ github.event_name != 'release' }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-20.04
          # - macos-12
        python-version: ["3.9"]
        dependencies:
          - pinned
          - fresh
        include:
          - os: ubuntu-20.04
            path: ~/.cache/pip
          # - os: macos-12
          # path: ~/Library/Caches/pip
        pytest-split-group: [1, 2, 3, 4, 5, 6] # Range the parallel test groups defined by env.PYTEST_SPLIT_GROUPS

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Cache pinned dependencies
        if: ${{ matrix.dependencies == 'pinned' }}
        uses: actions/cache@v3
        with:
          path: ${{ matrix.path }}
          key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-${{ hashFiles(format('./ci/requirements-py{0}.txt', matrix.python-version)) }}
          restore-keys: |
            ${{ runner.os }}-py${{ matrix.python-version }}-pip-

      # This step ensures that every month we have a new Pip cache, so that
      # we don't have one cache sitting for months growing ever-staler.
      - name: Get month for cache key
        id: month
        if: ${{ matrix.dependencies == 'fresh' }}
        run: |
          date +'cache-month=%Y-%m' >> "$GITHUB_ENV"

      - name: Cache fresh install dependencies
        if: ${{ matrix.dependencies == 'fresh' }}
        uses: actions/cache@v3
        with:
          path: ${{ matrix.path }}
          key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-fresh-${{ env.cache-month }}
          restore-keys: |
            ${{ runner.os }}-py${{ matrix.python-version }}-pip-fresh-
            ${{ runner.os }}-py${{ matrix.python-version }}-pip-

      - name: Install prerequisites
        run: |
          pip install --upgrade \
            pip \
            wheel \
            pytest \
            pytest-cov \
            pytest-explicit \
            pytest-split \
            setuptools \
            setuptools-scm

      - name: Install pinned dependencies
        if: ${{ matrix.dependencies == 'pinned' }}
        run: |
          pip install -r ci/requirements-py${{ matrix.python-version }}.txt

      - name: Install TileDB-Cloud-Py
        run: |
          pip install .[tests]
          pip install tiledb-vector-search --no-deps

      # Tries to restore the .test_durations file from the cache to use for splitting tests into groups.
      - name: "Attempt to restore .test_durations from cache"
        id: test-duration-cache-restore
        uses: actions/cache/restore@v4
        continue-on-error: true
        with:
          path: .test_durations
          key: ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}-${{ hashFiles('.test_durations') }}
          restore-keys: |
            ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}-

      - name: "Check .test_durations file"
        run: |
          if [ -f .test_durations ]; then
            echo ".test_durations file exists."
            file_content=$(cat .test_durations)
            echo $file_content
            if [ "$file_content" = "null" ]; then
              echo "Removing 'null' .test_durations file."
              rm .test_durations
            fi
          else
            echo ".test_durations file does not exist."
          fi

      - name: Run tests for vendored cloudpickle
        # Test pinned dependencies on commit / tag and fresh installs on nightlies
        if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
        run: |
          pip install psutil
          pip install src/tiledb/cloud/_vendor/cloudpickle/tests/cloudpickle_testpkg
          pytest -sv src/tiledb/cloud/_vendor/cloudpickle

      - name: Run tests
        # Test pinned dependencies on commit / tag and fresh installs on nightlies
        if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
        run: |
          pytest -sv \
            --tb short \
            --color yes \
            --splits ${{ env.PYTEST_SPLIT_GROUPS }} \
            --group ${{ matrix.pytest-split-group }} \
            --store-durations \
            --splitting-algorithm least_duration \
            --clean-durations \
            -r fExX \
          || .github/scripts/retry 2 \
            pytest -sv \
            --last-failed \
            --tb short \
            --color yes \
            -r fExX
        env:
          TILEDB_REST_TOKEN: ${{ secrets.TILEDB_CLOUD_HELPER_VAR }}

      # Uploads the partial durations from one of the run (specifically from the ubuntu run but it would be the same with macos).
      # These partial duration files will be combined into a single .test_durations file and used to update the test cache for the next runs
      - name: "Upload partial durations"
        uses: actions/upload-artifact@v4
        continue-on-error: true
        # Ensure that we save the partial test durations from an ubuntu job that the tests actually run.
        if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.9' && (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
        with:
          name: partial-test-durations-${{ matrix.pytest-split-group }}
          path: .test_durations
          retention-days: 1
          overwrite: true

  # Download and combine the partial test durations uploaded by the run-tests job,
  # into a single file and save it to cache for future use.
  save-test-durations-to-cache:
    name: Download, combine and save test_durations to the actions cache
    runs-on: ubuntu-latest
    needs: run-tests
    continue-on-error: true
    env:
      FolderPrefix: partial-test-durations
    steps:
      - name: Download partial durations
        uses: actions/download-artifact@v4
        with:
          # Download partial test duration artifacts in the same folder
          path: partial-test-durations
          pattern: partial-test-durations-*

      - name: "Combine test duration cache files in .test_durations"
        run: |
          jq -s add $(find partial-test-durations -type f -name .test_durations) > .test_durations
          cat .test_durations

      - name: "Save combined test durations to cache"
        uses: actions/cache/save@v4
        with:
          path: .test_durations
          key: ${{ env.PYTEST_TEST_DURATIONS_CACHE_KEY }}-${{ hashFiles('.test_durations') }}

  upload-to-pypi:
    # Upload a wheel only if the entire matrix succeeds.
    needs:
      - format-check
    # Run on release, or on commits to the primary branch.
    # Artifacts are uploaded to the real PyPI only when a release is published;
    # otherwise, they are sent just to the test instance.
    if: >
      (github.event_name == 'release' && github.event.action == 'published' && ${{ startsWith(github.event.release.tag_name, 'v') }}) ||
      (github.event_name == 'push')

    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          cache: pip
          cache-dependency-path: ci/requirements-py3.9.txt

      - name: Make fake version for Test PyPI
        if: ${{ github.event_name != 'release' }}
        run: |
          date -u +'SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0.dev%Y%m%d%H%M' >> "$GITHUB_ENV"

      - name: Build wheel
        run: |
          pip install --upgrade pip build
          python -m build

      - name: Publish to PyPI
        if: ${{ github.event_name == 'release' }}
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}

      - name: Publish to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
          user: __token__
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          # It's OK if this fails due to an already-existing file,
          # if the test publish was run more than once for a revision.
          skip-existing: true