Skip to content

Commit

Permalink
decrease docker images by utilizing multistage building (#12065)
Browse files Browse the repository at this point in the history
  • Loading branch information
colesnodgrass committed Apr 12, 2024
1 parent a4cb56a commit cd59733
Show file tree
Hide file tree
Showing 21 changed files with 100 additions and 99 deletions.
16 changes: 8 additions & 8 deletions airbyte-api-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE} AS server
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM ${JDK_IMAGE}
EXPOSE 8006 5005
ENV APPLICATION airbyte-api-server
ENV VERSION ${VERSION}

WORKDIR /app

# This is automatically unzipped by Docker
USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

# wait for upstream dependencies to become available before starting server
Expand Down
11 changes: 6 additions & 5 deletions airbyte-bootloader/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE}
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app

USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app

FROM ${JDK_IMAGE}
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-bootloader"]
File renamed without changes.
15 changes: 8 additions & 7 deletions airbyte-connector-sidecar/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.2.0

FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}

ARG DOCKER_BUILD_ARCH=amd64
Expand All @@ -9,12 +14,8 @@ ARG VERSION=dev
ENV APPLICATION airbyte-connector-sidecar
ENV VERSION=${VERSION}

USER root
COPY WellKnownTypes.json /app

# Move connector-sidecar app
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
COPY --chown=airbyte:airbyte WellKnownTypes.json /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

# wait for upstream dependencies to become available before starting server
Expand Down
16 changes: 8 additions & 8 deletions airbyte-container-orchestrator/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.2.0

FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}

# Don't change this manually. Bump version expects to make moves based on this string
Expand All @@ -8,13 +13,8 @@ ENV APPLICATION airbyte-container-orchestrator
ENV VERSION=${VERSION}

WORKDIR /app

USER root
COPY WellKnownTypes.json /app

# Move orchestrator app
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
COPY --chown=airbyte:airbyte WellKnownTypes.json /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

# wait for upstream dependencies to become available before starting server
Expand Down
11 changes: 6 additions & 5 deletions airbyte-cron/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE}
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app

USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app

FROM ${JDK_IMAGE}
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-cron"]
11 changes: 6 additions & 5 deletions airbyte-keycloak-setup/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE} AS keycloak-setup
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app

USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app

FROM ${JDK_IMAGE} AS keycloak-setup
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-keycloak-setup"]
20 changes: 9 additions & 11 deletions airbyte-keycloak/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,20 @@ COPY bin/scripts/entrypoint.sh entrypoint.sh
COPY bin/themes themes

# Doing this instead of creating a separate file and copying it to ensure that we get any keycloak updates to this conf file.
RUN cp conf/cache-ispn.xml conf/cache-ispn-override.xml
RUN sed -i conf/cache-ispn-override.xml -e 's/<distributed-cache name=\"sessions\" owners=\"2\">/<distributed-cache name=\"sessions\" owners=\"3\">/g'
RUN sed -i conf/cache-ispn-override.xml -e 's/<distributed-cache name=\"authenticationSessions\" owners=\"2\">/<distributed-cache name=\"authenticationSessions\" owners=\"3\">/g'

RUN cp conf/cache-ispn.xml conf/cache-ispn-override.xml && \
sed -i conf/cache-ispn-override.xml -e 's/<distributed-cache name=\"sessions\" owners=\"2\">/<distributed-cache name=\"sessions\" owners=\"3\">/g' && \
sed -i conf/cache-ispn-override.xml -e 's/<distributed-cache name=\"authenticationSessions\" owners=\"2\">/<distributed-cache name=\"authenticationSessions\" owners=\"3\">/g' && \
# Make sure that the two lines we wanted to be there are actually there
# i.e. keycloak didn't change its config file
RUN grep '<distributed-cache name="sessions" owners="3">' conf/cache-ispn-override.xml -q
RUN grep '<distributed-cache name="authenticationSessions" owners="3">' conf/cache-ispn-override.xml -q

grep '<distributed-cache name="sessions" owners="3">' conf/cache-ispn-override.xml -q && \
grep '<distributed-cache name="authenticationSessions" owners="3">' conf/cache-ispn-override.xml -q && \
# Create the directory for the infinispan global-state persistence
RUN mkdir -p /opt/keycloak/data/infinispan

mkdir -p /opt/keycloak/data/infinispan && \
# Inserting the <global-state> block after the <cache-container> start tag
RUN sed -i '/<cache-container /a \ <global-state><persistent-location path="\/opt\/keycloak\/data\/infinispan"\/><\/global-state>' conf/cache-ispn-override.xml
sed -i '/<cache-container /a \ <global-state><persistent-location path="\/opt\/keycloak\/data\/infinispan"\/><\/global-state>' conf/cache-ispn-override.xml && \
# Make sure that the <global-state> block is actually there
# i.e. keycloak didn't change its config file
RUN grep '<global-state><persistent-location path="/opt/keycloak/data/infinispan"/></global-state>' conf/cache-ispn-override.xml -q
grep '<global-state><persistent-location path="/opt/keycloak/data/infinispan"/></global-state>' conf/cache-ispn-override.xml -q


ENTRYPOINT ["./entrypoint.sh"]
File renamed without changes.
11 changes: 6 additions & 5 deletions airbyte-metrics/reporter/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE}
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app

USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app

FROM ${JDK_IMAGE}
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-metrics-reporter"]
File renamed without changes.
File renamed without changes.
17 changes: 7 additions & 10 deletions airbyte-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE} AS server
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

EXPOSE 8000 5005
FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM ${JDK_IMAGE} AS server
EXPOSE 8000 5005
ARG VERSION=dev

ENV APPLICATION airbyte-server
ENV VERSION ${VERSION}

WORKDIR /app

# This is automatically unzipped by Docker
USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

# wait for upstream dependencies to become available before starting server
Expand Down
File renamed without changes.
File renamed without changes.
8 changes: 6 additions & 2 deletions airbyte-webapp/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,12 @@ EXPOSE 8080
USER root

COPY bin/build /usr/share/nginx/html
RUN find /usr/share/nginx/html -type d -exec chmod 755 '{}' \; -o -type f -exec chmod 644 '{}' \;
RUN chown -R nginx:nginx /usr/share/nginx/html

RUN <<EOF
find /usr/share/nginx/html -type d -exec chmod 755 '{}' \; -o -type f -exec chmod 644 '{}' \;
chown -R nginx:nginx /usr/share/nginx/html
EOF

COPY bin/nginx/default.conf.template /etc/nginx/templates/default.conf.template

USER nginx:nginx
21 changes: 9 additions & 12 deletions airbyte-workers/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0
FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.2.0

FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}
ENV APPLICATION airbyte-workers
ENV VERSION ${VERSION}

EXPOSE 5005
WORKDIR /app

USER root
COPY WellKnownTypes.json /app

# Move worker app
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
COPY --chown=airbyte:airbyte WellKnownTypes.json /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

EXPOSE 5005

# wait for upstream dependencies to become available before starting server
ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"]
17 changes: 9 additions & 8 deletions airbyte-workload-api-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
FROM ${JDK_IMAGE} AS server
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1

FROM scratch as builder
WORKDIR /app
ADD airbyte-app.tar /app

FROM ${JDK_IMAGE}
EXPOSE 8007 5005
ENV APPLICATION airbyte-workload-api-server
ENV VERSION ${VERSION}
WORKDIR /app

# This is automatically unzipped by Docker
USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

# wait for upstream dependencies to become available before starting server
ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"]
21 changes: 10 additions & 11 deletions airbyte-workload-launcher/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0
FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}

ENV APPLICATION airbyte-workload-launcher
ENV VERSION ${VERSION}
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.2.0

FROM scratch as builder
WORKDIR /app

# This is automatically unzipped by Docker
USER root
ADD airbyte-app.tar /app
RUN chown -R airbyte:airbyte /app
USER airbyte:airbyte

FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}

ENV APPLICATION airbyte-workload-launcher
ENV VERSION ${VERSION}
# 8016 is the port micronaut listens on
# 5005 is the remote debug port
EXPOSE 8016 5005

# wait for upstream dependencies to become available before starting server
WORKDIR /app
COPY --chown=airbyte:airbyte --from=builder /app /app
USER airbyte:airbyte

ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"]
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ subprojects { sp ->
def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'

// Used by the platform -- Must be an Amazon Corretto-based image for build to work without modification to Dockerfile instructions
def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:3.2.0'
def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:3.2.1'

platform = buildPlatform
images.add("airbyte/$sp.dockerImageName:$rootProject.ext.image_tag")
Expand Down
2 changes: 1 addition & 1 deletion docker/airbyte-base-java-python-image/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.0
ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.2.1
FROM ${JDK_IMAGE}

USER root
Expand Down

0 comments on commit cd59733

Please sign in to comment.