From 6b0d8ed265bf7dddc04487df6dd1d788184cb7fa Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 1 Feb 2024 18:04:23 -0800 Subject: [PATCH] feat: docker image tags documentation + tweaks (#26923) Co-authored-by: Sam Firke --- .github/workflows/docker-release.yml | 40 ++++++++++++-- .github/workflows/ephemeral-env.yml | 10 +--- .github/workflows/no-op.yml | 14 ----- docker-compose-non-dev.yml | 2 +- docs/README.md | 5 +- docs/docs/installation/docker.mdx | 66 ++++++++++++++++++++++++ helm/superset/Chart.yaml | 2 +- helm/superset/README.md | 4 +- helm/superset/values.yaml | 2 +- scripts/build_docker.py | 25 +++++++-- tests/unit_tests/scripts/docker_build.py | 14 +++-- 11 files changed, 145 insertions(+), 39 deletions(-) create mode 100644 docs/docs/installation/docker.mdx diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index fe355c54dc50e..0d0770db28d5f 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -1,8 +1,26 @@ -name: Docker +name: Docker Publish Release on: release: types: [published] + + # Can be triggered manually + workflow_dispatch: + inputs: + release: + required: true + description: The version to generate + git-ref: + required: true + description: The git reference to checkout prior to running the docker build + force-latest: + required: true + type: choice + default: 'false' + description: Whether to force a latest tag on the release + options: + - true + - false jobs: config: runs-on: "ubuntu-latest" @@ -53,6 +71,22 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | - GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}" + RELEASE="${{ github.event.release.tag_name }}" + FORCE_LATEST="" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + # in the case of a manually-triggered run, read release from input + RELEASE ="${{ github.event.inputs.release }}" + if [ "${{ github.event.inputs.force-latest }}" = "true" ]; then + FORCE_LATEST="--force-latest" + fi + # build_docker.py may not exist on that SHA, let's switcharoo to /tmp + cp ./scripts/build_docker.py /tmp + git checkout "${{ github.event.inputs.git-ref }}" + cp /tmp/build_docker.py scripts/ + fi pip install click - ./scripts/build_docker.py ${{ matrix.build_preset }} release --platform ${{ matrix.platform }} + ./scripts/build_docker.py \ + ${{ matrix.build_preset }} \ + "${{ github.event_name }}" \ + --build_context_ref "$RELEASE" \ + --platform ${{ matrix.platform }} $FORCE_LATEST diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 5c944554407a0..0ba23e48acb5f 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -122,13 +122,7 @@ jobs: - name: Build ephemeral env image run: | - docker buildx build --target ci \ - --load \ - -t ${{ steps.get-sha.outputs.sha }} \ - -t "pr-${{ github.event.issue.number }}" \ - --platform linux/amd64 \ - --label "build_actor=${GITHUB_ACTOR}" \ - . + ./scripts/build_docker.py "ci" "pull_request" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -147,7 +141,7 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: superset-ci SHA: ${{ steps.get-sha.outputs.sha }} - IMAGE_TAG: pr-${{ github.event.issue.number }} + IMAGE_TAG: pr-${{ github.event.issue.number }}-ci run: | docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$SHA diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 4fecc2ba94030..835063640aa96 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -52,17 +52,3 @@ jobs: run: | echo "This is a no-op step for python-lint to ensure a successful status." exit 0 - check: - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for frontend-build to ensure a successful status." - exit 0 - docker-build: - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for frontend-build to ensure a successful status." - exit 0 diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index cf36ae833dd21..34aec9bbb7531 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev} +x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest} x-superset-depends-on: &superset-depends-on - db - redis diff --git a/docs/README.md b/docs/README.md index 1e6107564ae5c..cccd379862d09 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,4 +16,7 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -This is the public documentation site for Superset, built using [Docusaurus 2](https://docusaurus.io/). See [CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on contributing to documentation. +This is the public documentation site for Superset, built using +[Docusaurus 2](https://docusaurus.io/). See +[CONTRIBUTING.md](../CONTRIBUTING.md#documentation) for documentation on +contributing to documentation. diff --git a/docs/docs/installation/docker.mdx b/docs/docs/installation/docker.mdx new file mode 100644 index 0000000000000..fc6ca041235d5 --- /dev/null +++ b/docs/docs/installation/docker.mdx @@ -0,0 +1,66 @@ +# Docker Images and Tags + +The Apache Superset community extensively uses Docker for development, release, +and productionizing Superset. This page details our Docker builds and tag naming +schemes to help users navigate our offerings. + +Images are built and pushed to the [Superset Docker Hub repository]( +https://hub.docker.com/r/apache/superset). Different sets of images are created for: +- Published releases, with tags like `3.0.0` and the `latest` tag. +- Pull request iterations, each identified by tags starting with a SHA like + `8a2f7d378ab13c156fa183d9284b607ed69f5ecc`, and `pr-3454`, referencing the pull + request ID. +- Merges to the main branch (`master`), resulting in new SHAs, with tags + prefixed with `master` for the latest `master` version. + +Each code version has multiple builds for different purposes, identified by suffixes: +- **Build preset:** We offer various images for different needs: + - `lean`: The default Docker image, including both frontend and backend. Tags + without a build_preset are lean builds, e.g., `latest`. + - `dev`: For development, with a headless browser and root access. + - `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10). + - `ci`: For certain CI workloads. + - `websocket`: For Superset clusters supporting advanced features. + - `dockerize`: Used by Helm. +- **Platform:** We build for `linux/arm64` and `linux/amd64`. The `-arm` suffix + indicates ARM builds (e.g., `latest-arm`), while tags without a suffix are for + AMD (e.g., `latest`). + +## Key Image Tags and Examples + +- `latest`: The latest official release build, implicitly the lean build on + `linux/amd64`. +- `latest-dev`: the `-dev` image of the latest official release build, with a headless browser and root access. +- `master`: The latest build from the `master` branch, implicitly lean on + `linux/amd64`. +- `master-dev`: Similar to `master` but includes a headless browser and root access. +- `pr-5252`: The latest commit in PR 5252. +- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev-arm`: A `linux/arm64` build for + this specific SHA, which could be from a pull request, master merge, or release. +- `30948dc-dev-arm`: Same as above, but SHA truncated to 7 characters for a + shorter handle on the same image +- `websocket-latest`: The WebSocket image for use in a Superset cluster. + +For insights or modifications to the build matrix and tagging conventions, +check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py) +script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml) +GitHub action. + +## Caching + +To accelerate builds, we follow Docker best practices and use `apache/superset-cache`. + +## On ARM builds and working with Apple silicon + +Apple's current generation of computers uses ARM-based CPUs, and Docker +running on MACs seem to require `linux/arm64/v8` (at least one user's M2 was +configured in that way). Setting the environment +variable `DOCKER_DEFAULT_PLATFORM` to `linux/amd64` seems to function in +term of leveraging, and building upon the Superset builds provided here. + +``` +export DOCKER_DEFAULT_PLATFORM=linux/amd64 +``` + +Presumably, `linux/arm64/v8` would be more optimized for this generation +of chips, but less compatible across the ARM ecosystem. diff --git a/helm/superset/Chart.yaml b/helm/superset/Chart.yaml index 25ff660e38fc2..6926f3fcbbc0f 100644 --- a/helm/superset/Chart.yaml +++ b/helm/superset/Chart.yaml @@ -29,7 +29,7 @@ maintainers: - name: craig-rueda email: craig@craigrueda.com url: https://github.com/craig-rueda -version: 0.12.1 +version: 0.12.2 dependencies: - name: postgresql version: 12.1.6 diff --git a/helm/superset/README.md b/helm/superset/README.md index 27c1232440296..ed93586bc9ae9 100644 --- a/helm/superset/README.md +++ b/helm/superset/README.md @@ -23,7 +23,7 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs # superset -![Version: 0.12.1](https://img.shields.io/badge/Version-0.12.1-informational?style=flat-square) +![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square) Apache Superset is a modern, enterprise-ready business intelligence web application @@ -77,7 +77,7 @@ On helm this can be set on `extraSecretEnv.SUPERSET_SECRET_KEY` or `configOverri | hostAliases | list | `[]` | Custom hostAliases for all superset pods # https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"apachesuperset.docker.scarf.sh/apache/superset"` | | -| image.tag | string | `""` | | +| image.tag | string | `"latest"` | | | imagePullSecrets | list | `[]` | | | ingress.annotations | object | `{}` | | | ingress.enabled | bool | `false` | | diff --git a/helm/superset/values.yaml b/helm/superset/values.yaml index 15c5f7e214df4..3d96009d80103 100644 --- a/helm/superset/values.yaml +++ b/helm/superset/values.yaml @@ -179,7 +179,7 @@ extraConfigMountPath: "/app/configs" image: repository: apachesuperset.docker.scarf.sh/apache/superset - tag: "" + tag: "latest" pullPolicy: IfNotPresent imagePullSecrets: [] diff --git a/scripts/build_docker.py b/scripts/build_docker.py index f2323eb7eddc8..de6b8444f8ef8 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -83,6 +83,7 @@ def get_docker_tags( sha: str, build_context: str, build_context_ref: str, + force_latest: bool = False, ) -> set[str]: """ Return a set of tags given a given build context @@ -110,11 +111,13 @@ def get_docker_tags( if build_context == "release": # add a release tag tags.add(make_docker_tag([build_context_ref] + tag_chunks)) - if is_latest: + if is_latest or force_latest: # add a latest tag tags.add(make_docker_tag(["latest"] + tag_chunks)) elif build_context == "push" and build_context_ref == "master": tags.add(make_docker_tag(["master"] + tag_chunks)) + elif build_context == "pull_request": + tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks)) return tags @@ -125,6 +128,7 @@ def get_docker_command( sha: str, build_context: str, build_context_ref: str, + force_latest: bool = False, ) -> str: tag = "" build_target = "" @@ -160,6 +164,7 @@ def get_docker_command( sha, build_context, build_context_ref, + force_latest, ) docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags]) @@ -205,12 +210,16 @@ def get_docker_command( ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") +@click.option( + "--force-latest", is_flag=True, help="Force the 'latest' tag on the release" +) def main( build_preset: str, build_context: str, build_context_ref: str, platform: str, dry_run: bool, + force_latest: bool, ) -> None: """ This script executes docker build and push commands based on given arguments. @@ -219,7 +228,16 @@ def main( is_authenticated = ( True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False ) - build_context_ref = get_build_context_ref(build_context) + + if force_latest and build_context != "release": + print( + "--force-latest can only be applied if the build context is set to 'release'" + ) + exit(1) + + if build_context == "release" and not build_context_ref.strip(): + print("Release number has to be provided") + exit(1) docker_build_command = get_docker_command( build_preset, @@ -227,7 +245,8 @@ def main( is_authenticated, get_git_sha(), build_context, - get_build_context_ref(build_context), + build_context_ref, + force_latest, ) if not dry_run: diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py index 002a51f27dc14..ee9ad66ead212 100644 --- a/tests/unit_tests/scripts/docker_build.py +++ b/tests/unit_tests/scripts/docker_build.py @@ -65,7 +65,7 @@ def test_is_latest_release(release, expected_bool): SHA, "pull_request", PR_ID, - [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"], + [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"], ), ( "ci", @@ -73,7 +73,7 @@ def test_is_latest_release(release, expected_bool): SHA, "pull_request", PR_ID, - [f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci"], + [f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"], ), ( "lean", @@ -81,7 +81,7 @@ def test_is_latest_release(release, expected_bool): SHA, "pull_request", PR_ID, - [f"{REPO}:22e7c60", f"{REPO}:{SHA}"], + [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"], ), ( "dev", @@ -89,7 +89,11 @@ def test_is_latest_release(release, expected_bool): SHA, "pull_request", PR_ID, - [f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"], + [ + f"{REPO}:22e7c60-dev-arm", + f"{REPO}:{SHA}-dev-arm", + f"{REPO}:pr-{PR_ID}-dev-arm", + ], ), ( "dev", @@ -97,7 +101,7 @@ def test_is_latest_release(release, expected_bool): SHA, "pull_request", PR_ID, - [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"], + [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"], ), # old releases (