Add PRIVILEGED_NESTED to the buildah task

Signed-off-by: Wei Shi <[email protected]>
konflux-ci · Nov 12, 2024 · 8670c64 · 8670c64
1 parent 761bdc3
commit 8670c64
Show file tree

Hide file tree

Showing 8 changed files with 115 additions and 1 deletion.
diff --git a/pipelines/docker-build-multi-platform-oci-ta/README.md b/pipelines/docker-build-multi-platform-oci-ta/README.md
@@ -61,6 +61,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |LABELS| Additional key=value labels that should be applied to the image| []| |
 |PLATFORM| The platform to build on| None| |
 |PREFETCH_INPUT| In case it is not empty, the prefetched content should be made available to the build.| | '$(params.prefetch-input)'|
+|PRIVILEGED_NESTED| Whether to enable privileged mode, it will add --privileged to podman run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah build| false| |
 |SKIP_UNUSED_STAGES| Whether to skip stages in Containerfile that seem unused by subsequent stages| true| |
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)'|
 |SQUASH| Squash all new and previous layers added as a part of this build, as per --squash| false| |

diff --git a/task-generator/remote/main.go b/task-generator/remote/main.go
@@ -164,6 +164,7 @@ if ! [[ $IS_LOCALHOST ]]; then
   export BUILD_DIR=$(cat /ssh/user-dir)
   export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
   echo "$BUILD_DIR"
+  # shellcheck disable=SC2086
   ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
   PORT_FORWARD=""
@@ -234,13 +235,23 @@ if ! [[ $IS_LOCALHOST ]]; then
 			}
 		}
 		ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
+		ret += "\n"
+		ret += `  PRIVILEGED_NESTED_FLAGS=()
+  if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+    # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+    # https://github.com/coreos/rpm-ostree/discussions/4648
+    # shellcheck disable=SC2086
+    ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+    PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+  fi`
 		ret += "\n  rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
 		containerScript := "scripts/script-" + step.Name + ".sh"
 		for _, e := range step.Env {
 			env += "    -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
 		}
 		podmanArgs += "    -v \"$BUILD_DIR/scripts:/scripts:Z\" \\\n"
-		ret += "\n  ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "    --user=0  --rm  \"$BUILDER_IMAGE\" /" + containerScript + ` "$@"`
+		podmanArgs += "    \"${PRIVILEGED_NESTED_FLAGS[@]}\" \\\n"
+		ret += "\n  # shellcheck disable=SC2086\n  ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "    --user=0  --rm  \"$BUILDER_IMAGE\" /" + containerScript + ` "$@"`
 
 		// Sync the contents of the workspaces back so subsequent tasks can use them
 		for _, workspace := range task.Spec.Workspaces {
@@ -293,6 +304,8 @@ echo $container > /shared/container_name`
 		},
 	})
 	task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "BUILDER_IMAGE", Value: builderImage})
+	task.Spec.Params = append(task.Spec.Params, tektonapi.ParamSpec{Name: "PRIVILEGED_NESTED", Type: tektonapi.ParamTypeString, Description: "Whether to enable privileged mode, it will add --privileged to podman run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah build", Default: &tektonapi.ParamValue{StringVal: "false", Type: tektonapi.ParamTypeString}})
+	task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "PRIVILEGED_NESTED", Value: "$(params.PRIVILEGED_NESTED)"})
 	if taskVersion != "0.1" {
 		task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "PLATFORM", Value: "$(params.PLATFORM)"})
 

diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
@@ -372,6 +372,12 @@ spec:
 
         BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+        if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+          BUILDAH_ARGS+=("--security-opt=label=disable")
+          BUILDAH_ARGS+=("--cap-add=all")
+          BUILDAH_ARGS+=("--device=/dev/fuse")
+        fi
+
         if [ -n "${ADD_CAPABILITIES}" ]; then
           BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
         fi

diff --git a/task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml
@@ -132,6 +132,12 @@ spec:
   - description: The platform to build on
     name: PLATFORM
     type: string
+  - default: "false"
+    description: Whether to enable privileged mode, it will add --privileged to podman
+      run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah
+      build
+    name: PRIVILEGED_NESTED
+    type: string
   results:
   - description: Digests of the base images used for build
     name: BASE_IMAGES_DIGESTS
@@ -186,6 +192,8 @@ spec:
       value: $(params.YUM_REPOS_D_TARGET)
     - name: BUILDER_IMAGE
       value: quay.io/konflux-ci/buildah-task:latest@sha256:5cbd487022fb7ac476cbfdea25513b810f7e343ec48f89dc6a4e8c3c39fa37a2
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     volumeMounts:
     - mountPath: /shared
       name: shared
@@ -241,6 +249,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -446,7 +455,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
@@ -477,6 +495,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/

diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml
@@ -131,6 +131,12 @@ spec:
   - description: The platform to build on
     name: PLATFORM
     type: string
+  - default: "false"
+    description: Whether to enable privileged mode, it will add --privileged to podman
+      run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah
+      build
+    name: PRIVILEGED_NESTED
+    type: string
   - default: "false"
     description: Whether to append a sanitized platform architecture on the IMAGE
       tag
@@ -196,6 +202,8 @@ spec:
       value: $(params.YUM_REPOS_D_TARGET)
     - name: BUILDER_IMAGE
       value: quay.io/konflux-ci/buildah-task:latest@sha256:b2d6c32d1e05e91920cd4475b2761d58bb7ee11ad5dff3ecb59831c7572b4d0c
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     - name: PLATFORM
       value: $(params.PLATFORM)
     - name: IMAGE_APPEND_PLATFORM
@@ -261,6 +269,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -406,6 +415,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi
@@ -575,7 +590,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e ACTIVATION_KEY="$ACTIVATION_KEY" \
@@ -608,6 +632,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/

diff --git a/task/buildah-remote/0.1/buildah-remote.yaml b/task/buildah-remote/0.1/buildah-remote.yaml
@@ -127,6 +127,12 @@ spec:
   - description: The platform to build on
     name: PLATFORM
     type: string
+  - default: "false"
+    description: Whether to enable privileged mode, it will add --privileged to podman
+      run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah
+      build
+    name: PRIVILEGED_NESTED
+    type: string
   results:
   - description: Digest of the image just built
     name: IMAGE_DIGEST
@@ -183,6 +189,8 @@ spec:
       value: $(params.SKIP_UNUSED_STAGES)
     - name: BUILDER_IMAGE
       value: quay.io/konflux-ci/buildah-task:latest@sha256:5cbd487022fb7ac476cbfdea25513b810f7e343ec48f89dc6a4e8c3c39fa37a2
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     volumeMounts:
     - mountPath: /shared
       name: shared
@@ -229,6 +237,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -438,7 +447,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
@@ -470,6 +488,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/source/" "$(workspaces.source.path)/"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/

diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml
@@ -122,6 +122,12 @@ spec:
   - description: The platform to build on
     name: PLATFORM
     type: string
+  - default: "false"
+    description: Whether to enable privileged mode, it will add --privileged to podman
+      run and --security-opt=label=disable --cap-add=all --device=/dev/fuse to buildah
+      build
+    name: PRIVILEGED_NESTED
+    type: string
   - default: "false"
     description: Whether to append a sanitized platform architecture on the IMAGE
       tag
@@ -187,6 +193,8 @@ spec:
       value: $(params.SKIP_UNUSED_STAGES)
     - name: BUILDER_IMAGE
       value: quay.io/konflux-ci/buildah-task:latest@sha256:b2d6c32d1e05e91920cd4475b2761d58bb7ee11ad5dff3ecb59831c7572b4d0c
+    - name: PRIVILEGED_NESTED
+      value: $(params.PRIVILEGED_NESTED)
     - name: PLATFORM
       value: $(params.PLATFORM)
     - name: IMAGE_APPEND_PLATFORM
@@ -243,6 +251,7 @@ spec:
         export BUILD_DIR=$(cat /ssh/user-dir)
         export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
         echo "$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
 
         PORT_FORWARD=""
@@ -382,6 +391,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi
@@ -552,7 +567,16 @@ spec:
       chmod +x scripts/script-build.sh
 
       if ! [[ $IS_LOCALHOST ]]; then
+        PRIVILEGED_NESTED_FLAGS=()
+        if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then
+          # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes.
+          # https://github.com/coreos/rpm-ostree/discussions/4648
+          # shellcheck disable=SC2086
+          ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/var/tmp"
+          PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared")
+        fi
         rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        # shellcheck disable=SC2086
         ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
           --tmpfs /run/secrets \
           -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
@@ -585,6 +609,7 @@ spec:
           -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
           -v "$BUILD_DIR/results/:/tekton/results:Z" \
           -v "$BUILD_DIR/scripts:/scripts:Z" \
+          "${PRIVILEGED_NESTED_FLAGS[@]}" \
           --user=0  --rm  "$BUILDER_IMAGE" /scripts/script-build.sh "$@"
         rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/source/" "$(workspaces.source.path)/"
         rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/

diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml
@@ -303,6 +303,12 @@ spec:
 
       BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}")
 
+      if [ "${PRIVILEGED_NESTED}" == "true" ]; then
+        BUILDAH_ARGS+=("--security-opt=label=disable")
+        BUILDAH_ARGS+=("--cap-add=all")
+        BUILDAH_ARGS+=("--device=/dev/fuse")
+      fi
+
       if [ -n "${ADD_CAPABILITIES}" ]; then
         BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}")
       fi