diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 05120bb4bb4a..31cd1dd00ba1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -41,28 +41,38 @@ jobs: - name: Ensure no file change run: git diff-index --quiet HEAD - - name: Build Integration Docker Images (Master branch only) + - name: Build Integration Docker Images if: success() && github.ref == 'refs/heads/master' run: ./gradlew buildImage --no-daemon -g ${{ env.GRADLE_PATH }} - - name: Write BigQuery Integration Test Credentials (Master branch only) + - name: Write BigQuery Integration Test Credentials if: success() && github.ref == 'refs/heads/master' run: 'mkdir dataline-integrations/singer/bigquery/destination/config && echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > dataline-integrations/singer/bigquery/destination/config/credentials.json' env: - BIGQUERY_INTEGRATION_TEST_CREDS: ${{secrets.BIGQUERY_INTEGRATION_TEST_CREDS}} + BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} - - name: Run Integration Tests (Master branch only) + - name: Run Integration Tests if: success() && github.ref == 'refs/heads/master' run: ./gradlew integrationTest --no-daemon -g ${{ env.GRADLE_PATH }} - - name: Build Core Docker Images (Master branch only) + - name: Build Core Docker Images if: success() && github.ref == 'refs/heads/master' run: docker-compose -f docker-compose.build.yaml build + env: + GIT_REVISION: ${{ github.sha }} - name: Run End-to-End Acceptance Tests (Master branch only) if: success() && github.ref == 'refs/heads/master' run: ./tools/bin/acceptance_test.sh + - name: Push Core Docker Images + if: success() && github.ref == 'refs/heads/master' + run: | + docker login -u datalinebot -p ${DOCKER_PASSWORD} + docker-compose -f docker-compose.build.yaml push + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + - name: Slack Notification (Master branch only) if: failure() && github.ref == 'refs/heads/master' uses: rtCamp/action-slack-notify@master diff --git a/.gitignore b/.gitignore index 93fd4c323eec..ebddc29cee1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .gradle .idea build +!tools/build .DS_Store dataline-db/pg_data/* data diff --git a/Dockerfile.build b/Dockerfile.build index 9af3d8f9536e..212b244bf039 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -1,25 +1,7 @@ -##################### -# Setup build image # -##################### -FROM ubuntu:20.04 AS build-base - -WORKDIR /code - -ENV DEBIAN_FRONTEND noninteractive - -# Install tools -RUN apt-get update && apt-get -y install curl - -# Setup Node & Java -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - -RUN apt-get update && apt-get -y install \ - nodejs \ - openjdk-14-jdk - ####################### # Prepare project env # ####################### -FROM build-base AS build-project +FROM dataline/build-base:1.0.0 AS build-project WORKDIR /code diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index 0b8bbd16e4e4..6c275c9d86b7 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -6,31 +6,37 @@ services: build: dockerfile: Dockerfile.database context: . + labels: + io.dataline.git-revision: ${GIT_REVISION} seed: image: dataline/seed:dev build: dockerfile: Dockerfile.build target: seed context: . + labels: + io.dataline.git-revision: ${GIT_REVISION} scheduler: image: dataline/scheduler:dev build: dockerfile: Dockerfile.build target: scheduler context: . + labels: + io.dataline.git-revision: ${GIT_REVISION} server: image: dataline/server:dev build: dockerfile: Dockerfile.build target: server context: . + labels: + io.dataline.git-revision: ${GIT_REVISION} webapp: image: dataline/webapp:dev build: dockerfile: Dockerfile.build target: webapp context: . - -networks: - dataline: - driver: "bridge" + labels: + io.dataline.git-revision: ${GIT_REVISION} diff --git a/tools/build/Dockerfile.base b/tools/build/Dockerfile.base new file mode 100644 index 000000000000..adedb92e78ed --- /dev/null +++ b/tools/build/Dockerfile.base @@ -0,0 +1,20 @@ +##################### +# Setup build image # +##################### +FROM ubuntu:20.04 + +LABEL io.dataline.image=datalineio/build-base +LABEL io.dataline.version=1.0.0 + +WORKDIR /code + +ENV DEBIAN_FRONTEND noninteractive + +# Install tools +RUN apt-get update && apt-get -y install curl + +# Setup Node & Java +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - +RUN apt-get update && apt-get -y install \ + nodejs \ + openjdk-14-jdk diff --git a/tools/build/README.md b/tools/build/README.md new file mode 100644 index 000000000000..41aaf3fa6b37 --- /dev/null +++ b/tools/build/README.md @@ -0,0 +1,6 @@ +This docker image contains the build environment used to generate all our artifacts. + +If you need to do some changes the environment: +1. Bump up the version in the docker file (`io.dataline.version`). +1. Build and push the image (with the new tag). +1. Update the consumers to use the most recent build image.