Run benchmarks #512
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run benchmarks | |
| on: | |
| schedule: | |
| - cron: "0 0 */1 * *" | |
| workflow_dispatch: | |
| jobs: | |
| benchmark_single: | |
| runs-on: ubicloud-standard-8 | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_DB: windmill | |
| POSTGRES_PASSWORD: changeme | |
| options: >- | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 | |
| windmill: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --health-cmd "curl | |
| http://localhost:8000/api/version" | |
| ports: | |
| - 8000:8000 | |
| steps: | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: benchmark | |
| timeout-minutes: 20 | |
| run: deno run --unstable -A -r | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts | |
| -c | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json | |
| - name: Save benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_single | |
| path: | | |
| *.json | |
| benchmark_dedicated: | |
| runs-on: ubicloud-standard-8 | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_DB: windmill | |
| POSTGRES_PASSWORD: changeme | |
| options: >- | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 | |
| windmill: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| WORKER_GROUP: dedicated | |
| DEDICATED_WORKER: "admins:f/benchmarks/dedicated" | |
| options: >- | |
| --pull always --restart unless-stopped --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --health-cmd "curl | |
| http://localhost:8000/api/version" | |
| ports: | |
| - 8000:8000 | |
| steps: | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: benchmark | |
| timeout-minutes: 20 | |
| run: deno run --unstable -A -r | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts | |
| --no-warm-up -c | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json | |
| - name: Save benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_dedicated | |
| path: | | |
| *.json | |
| benchmark_4workers: | |
| runs-on: ubicloud-standard-8 | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_DB: windmill | |
| POSTGRES_PASSWORD: changeme | |
| options: >- | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 | |
| windmill: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --health-cmd "curl | |
| http://localhost:8000/api/version" | |
| ports: | |
| - 8000:8000 | |
| windmill_1: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_2: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_3: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| steps: | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: benchmark | |
| timeout-minutes: 20 | |
| run: deno run --unstable -A -r | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts | |
| -c | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json | |
| --workers 4 | |
| - name: Save benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_4workers | |
| path: | | |
| *.json | |
| benchmark_8workers: | |
| runs-on: ubicloud-standard-8 | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_DB: windmill | |
| POSTGRES_PASSWORD: changeme | |
| options: >- | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 | |
| windmill: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --health-cmd "curl | |
| http://localhost:8000/api/version" | |
| ports: | |
| - 8000:8000 | |
| windmill_1: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_2: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_3: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_4: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_5: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_6: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| windmill_7: | |
| image: ghcr.io/windmill-labs/windmill-ee:main | |
| env: | |
| DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| MODE: worker | |
| WORKER_GROUP: main | |
| WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets | |
| options: >- | |
| --pull always | |
| steps: | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: benchmark | |
| timeout-minutes: 20 | |
| run: deno run --unstable -A -r | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts | |
| -c | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json | |
| --workers 8 | |
| - name: Save benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_8workers | |
| path: | | |
| *.json | |
| benchmark_graphs: | |
| runs-on: ubicloud | |
| needs: | |
| - benchmark_single | |
| - benchmark_dedicated | |
| - benchmark_4workers | |
| - benchmark_8workers | |
| steps: | |
| - uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: benchmarks | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| merge-multiple: true | |
| - name: graphs | |
| run: deno run --unstable -A -r | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts | |
| -c | |
| https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json | |
| - name: Push changes | |
| run: | | |
| ls -la | |
| pwd | |
| git add . | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git commit -m "Update benchmarks" | |
| git push |