fix tests #9630
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| env: | |
| REGISTRY: ghcr.io | |
| ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com | |
| IMAGE_NAME: ${{ github.repository }} | |
| name: Build windmill:main | |
| on: | |
| push: | |
| branches: [main] | |
| tags: ["*"] | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| id-token: write | |
| packages: write | |
| jobs: | |
| build: | |
| runs-on: ubicloud | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Read EE repo commit hash | |
| run: | | |
| echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: windmill-labs/windmill-ee-private | |
| path: ./windmill-ee-private | |
| ref: ${{ env.ee_repo_ref }} | |
| token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} | |
| fetch-depth: 0 | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| - uses: depot/setup-action@v1 | |
| - name: Login to registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Substitute EE code (EE logic is behind feature flag) | |
| run: | | |
| ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private | |
| - name: Docker meta | |
| id: meta-public | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| flavor: | | |
| latest=false | |
| tags: | | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - name: Build and push publicly | |
| uses: depot/build-push-action@v1 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| build-args: | | |
| features=embedding,parquet | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| ${{ steps.meta-public.outputs.tags }} | |
| labels: | | |
| ${{ steps.meta-public.outputs.labels }} | |
| org.opencontainers.image.licenses=AGPLv3 | |
| build_ee: | |
| runs-on: ubicloud | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Read EE repo commit hash | |
| run: | | |
| echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: windmill-labs/windmill-ee-private | |
| path: ./windmill-ee-private | |
| ref: ${{ env.ee_repo_ref }} | |
| token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} | |
| fetch-depth: 0 | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| - uses: depot/setup-action@v1 | |
| - name: Docker meta | |
| id: meta-ee-public | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee | |
| flavor: | | |
| latest=false | |
| tags: | | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - name: Login to registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Substitute EE code | |
| run: | | |
| ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private | |
| - name: Build and push publicly ee | |
| uses: depot/build-push-action@v1 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| build-args: | | |
| features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus | |
| nsjail=true | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| ${{ steps.meta-ee-public.outputs.tags }} | |
| labels: | | |
| ${{ steps.meta-ee-public.outputs.labels }} | |
| org.opencontainers.image.licenses=Windmill-Enterprise-License | |
| build_ee_reports_privately: | |
| needs: [build_ee] | |
| runs-on: ubicloud | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| - uses: depot/setup-action@v1 | |
| - name: Docker meta | |
| id: meta-ee-public | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=sha,enable=true,priority=100,prefix=,suffix=,format=short | |
| - name: Login to ECR | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.ECR_REGISTRY }} | |
| username: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Build and push publicly ee reports | |
| uses: depot/build-push-action@v1 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: true | |
| file: "./docker/DockerfileReports" | |
| tags: | | |
| ${{ steps.meta-ee-public.outputs.tags }} | |
| labels: | | |
| ${{ steps.meta-ee-public.outputs.labels }} | |
| org.opencontainers.image.licenses=Windmill-Enterprise-License | |
| # disabled until we make it 100% reliable and add more meaningful tests | |
| # playwright: | |
| # runs-on: [self-hosted, new] | |
| # needs: [build] | |
| # services: | |
| # postgres: | |
| # image: postgres | |
| # env: | |
| # POSTGRES_DB: windmill | |
| # POSTGRES_USER: admin | |
| # POSTGRES_PASSWORD: changeme | |
| # ports: | |
| # - 5432:5432 | |
| # options: >- | |
| # --health-cmd pg_isready | |
| # --health-interval 10s | |
| # --health-timeout 5s | |
| # --health-retries 5 | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: "Docker" | |
| # run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)" | |
| # id: docker-container | |
| # - uses: actions/setup-node@v3 | |
| # with: | |
| # node-version: 16 | |
| # - name: "Playwright run" | |
| # timeout-minutes: 2 | |
| # run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test | |
| # - name: "Clean up" | |
| # run: docker kill ${{ steps.docker-container.outputs.id }} | |
| # if: always() | |
| attach_amd64_binary_to_release: | |
| needs: [build, build_ee] | |
| runs-on: ubicloud | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| env: | |
| ARCH: amd64 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| # pulling docker image with desired arch so that actions-docker-extract doesn't do it | |
| docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| - run: | | |
| # Checks the image is in docker prior to running actions-docker-extract. It fails if not | |
| # Also useful to visually check that the arch is the right opencontainers | |
| docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| - uses: shrink/actions-docker-extract@v3 | |
| id: extract | |
| with: | |
| image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| path: "/usr/src/app/windmill" | |
| - uses: shrink/actions-docker-extract@v3 | |
| id: extract-ee | |
| with: | |
| image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| path: "/usr/src/app/windmill" | |
| - name: Rename binary with corresponding architecture | |
| run: | | |
| mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}" | |
| mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}" | |
| - name: Attach binary to release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| ${{ steps.extract.outputs.destination }}/* | |
| ${{ steps.extract-ee.outputs.destination }}/* | |
| # attach_arm64_binary_to_release: | |
| # needs: [build, build_ee] | |
| # runs-on: ubicoud | |
| # if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| # env: | |
| # ARCH: arm64 | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - run: | | |
| # # pulling docker image with desired arch so that actions-docker-extract doesn't do it | |
| # docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| # docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| # - run: | | |
| # # Checks the image is in docker prior to running actions-docker-extract. It fails if not | |
| # # Also useful to visually check that the arch is the right opencontainers | |
| # docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| # docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| # - uses: shrink/actions-docker-extract@v3 | |
| # id: extract | |
| # with: | |
| # image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev | |
| # path: "/usr/src/app/windmill" | |
| # - uses: shrink/actions-docker-extract@v3 | |
| # id: extract-ee | |
| # with: | |
| # image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| # path: "/usr/src/app/windmill" | |
| # - name: Rename binary with corresponding architecture | |
| # run: | | |
| # mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}" | |
| # mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}" | |
| # - name: Attach binary to release | |
| # uses: softprops/action-gh-release@v1 | |
| # with: | |
| # files: | | |
| # ${{ steps.extract.outputs.destination }}/* | |
| # ${{ steps.extract-ee.outputs.destination }}/* | |
| publish_ecr_s3: | |
| needs: [build_ee] | |
| runs-on: ubicloud | |
| if: github.event_name != 'pull_request' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to ECR | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.ECR_REGISTRY }} | |
| username: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Push image to ECR | |
| if: github.event_name != 'pull_request' | |
| id: push_ecr | |
| run: | | |
| git_hash=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx imagetools create \ | |
| --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \ | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev | |
| echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT" | |
| - uses: shrink/actions-docker-extract@v3 | |
| if: github.event_name != 'pull_request' | |
| id: extract | |
| with: | |
| image: |- | |
| ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ steps.push_ecr.outputs.GIT_HASH }} | |
| path: "/static_frontend/." | |
| - uses: reggionick/s3-deploy@v3 | |
| if: github.event_name != 'pull_request' | |
| with: | |
| folder: ${{ steps.extract.outputs.destination }} | |
| bucket: windmill-frontend | |
| bucket-region: us-east-1 | |
| run_integration_test: | |
| runs-on: ubicloud | |
| needs: [build_ee] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Prepare test run | |
| if: ${{ ! startsWith(github.ref, 'refs/tags/') }} | |
| run: cd integration_tests && ./build.sh | |
| - name: Test run | |
| if: ${{ ! startsWith(github.ref, 'refs/tags/') }} | |
| timeout-minutes: 15 | |
| env: | |
| LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} | |
| run: cd integration_tests && ./run.sh | |
| - name: Archive logs | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Windmill Integration Tests Logs | |
| path: | | |
| integration_tests/logs | |
| tag_latest: | |
| runs-on: ubicloud | |
| needs: [run_integration_test] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Login to registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Tag main and latest | |
| run: | | |
| docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest | |
| docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main | |
| tag_latest_ee: | |
| runs-on: ubicloud | |
| needs: [run_integration_test] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Login to registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Tag main and latest for ee | |
| run: | | |
| docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest | |
| docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main | |
| build_ee_cuda: | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| needs: [build_ee] | |
| runs-on: ubicloud | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| - uses: depot/setup-action@v1 | |
| - name: Docker meta | |
| id: meta-ee-public | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda | |
| flavor: | | |
| latest=false | |
| tags: | | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - name: Login to registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push publicly ee | |
| uses: depot/build-push-action@v1 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: true | |
| file: "./docker/DockerfileCuda" | |
| tags: | | |
| ${{ steps.meta-ee-public.outputs.tags }} | |
| labels: | | |
| ${{ steps.meta-ee-public.outputs.labels }} | |
| org.opencontainers.image.licenses=Windmill-Enterprise-License |