diff --git a/.github/workflows/test.yaml b/.github/workflows/test_query.yaml similarity index 50% rename from .github/workflows/test.yaml rename to .github/workflows/test_query.yaml index 117c167fd7e0..c8127d54b43f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test_query.yaml @@ -1,12 +1,4 @@ -# We use self-hosted runners for performance: we can rely on docker caching -# on the runner machines for 0s startup times for our tests. -# There is an issue with self-hosted runners where if you use a mix of -# host and container jobs, the host jobs will fail to start -# (https://github.com/actions/checkout/issues/273). -# So we create a separate "builder" tagged runner to do the build job on -# the host, and then all the tests run on "runner" tagged runners. - -name: test +name: test_query on: push: @@ -134,117 +126,3 @@ jobs: yarn tslint yarn prettier yarn run tsc - - # ==== Trace Jobs ==== - lint: - name: Python lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install tox - - name: Run tox - run: tox -e lint - - trace-tests: - name: Trace tox tests - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - matrix: - python-version-major: ['3'] - python-version-minor: [ - '9', - '10', - '11', - '12', - # - ] - tox-shard: - [ - 'trace', - 'trace_server', - 'anthropic', - 'cerebras', - 'cohere', - 'dspy', - 'groq', - 'instructor', - 'langchain', - 'litellm', - 'llamaindex', - 'mistral0', - 'mistral1', - 'openai', - ] - fail-fast: false - services: - wandbservice: - image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master - credentials: - username: _json_key - password: ${{ secrets.gcp_wb_sa_key }} - env: - CI: 1 - WANDB_ENABLE_TEST_CONTAINER: true - ports: - - '8080:8080' - - '8083:8083' - - '9015:9015' - options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s - weave_clickhouse: - image: clickhouse/clickhouse-server - ports: - - '8123:8123' - options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install tox - - name: Run tox (Clickhouse Only) - env: - WEAVE_SENTRY_ENV: ci - CI: 1 - WB_SERVER_HOST: http://wandbservice - WF_CLICKHOUSE_HOST: localhost - WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 - run: | - tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} -- \ - -m "weave_client and not skip_clickhouse_client" \ - --weave-server=clickhouse - - name: Run tox - env: - WEAVE_SENTRY_ENV: ci - CI: 1 - WB_SERVER_HOST: http://wandbservice - WF_CLICKHOUSE_HOST: weave_clickhouse - WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 - run: | - tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} - trace-tests-matrix-check: # This job does nothing and is only used for the branch protection - if: always() - - needs: - - trace-tests - - runs-on: ubuntu-latest - - steps: - - name: Passes if all trace-tests jobs succeeded - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/test_trace.yaml b/.github/workflows/test_trace.yaml new file mode 100644 index 000000000000..757586dcb701 --- /dev/null +++ b/.github/workflows/test_trace.yaml @@ -0,0 +1,129 @@ +# We use self-hosted runners for performance: we can rely on docker caching +# on the runner machines for 0s startup times for our tests. +# There is an issue with self-hosted runners where if you use a mix of +# host and container jobs, the host jobs will fail to start +# (https://github.com/actions/checkout/issues/273). +# So we create a separate "builder" tagged runner to do the build job on +# the host, and then all the tests run on "runner" tagged runners. + +name: test_trace + +on: + push: + workflow_call: + +jobs: + determine_tests: + uses: ./github/workflows/which-tests-to-run.yaml + lint: + name: Python lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Run tox + run: tox -e lint + + trace-tests: + name: Trace tox tests + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + matrix: + python-version-major: ['3'] + python-version-minor: [ + '9', + '10', + '11', + '12', + # + ] + tox-shard: + [ + 'trace', + 'trace_server', + 'anthropic', + 'cerebras', + 'cohere', + 'dspy', + 'groq', + 'instructor', + 'langchain', + 'litellm', + 'llamaindex', + 'mistral0', + 'mistral1', + 'openai', + ] + fail-fast: false + services: + wandbservice: + image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master + credentials: + username: _json_key + password: ${{ secrets.gcp_wb_sa_key }} + env: + CI: 1 + WANDB_ENABLE_TEST_CONTAINER: true + ports: + - '8080:8080' + - '8083:8083' + - '9015:9015' + options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s + weave_clickhouse: + image: clickhouse/clickhouse-server + ports: + - '8123:8123' + options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Run tox (Clickhouse Only) + env: + WEAVE_SENTRY_ENV: ci + CI: 1 + WB_SERVER_HOST: http://wandbservice + WF_CLICKHOUSE_HOST: localhost + WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 + run: | + tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} -- \ + -m "weave_client and not skip_clickhouse_client" \ + --weave-server=clickhouse + - name: Run tox + env: + WEAVE_SENTRY_ENV: ci + CI: 1 + WB_SERVER_HOST: http://wandbservice + WF_CLICKHOUSE_HOST: weave_clickhouse + WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 + run: | + tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} + trace-tests-matrix-check: # This job does nothing and is only used for the branch protection + if: always() + + needs: + - trace-tests + + runs-on: ubuntu-latest + + steps: + - name: Passes if all trace-tests jobs succeeded + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }}