refactor(dockerfile): Remove stackrox-data Dockerfile (stackrox#6653)

.github/workflows/build.yaml

@@ -216,7 +216,7 @@ jobs:
         with:
           name: docs-build
           path: |
-            image/docs
+            image/rhel/docs
 
   build-and-push-main:
     runs-on: ubuntu-latest
@@ -318,7 +318,7 @@ jobs:
       - uses: actions/download-artifact@v3
         with:
           name: docs-build
-          path: image/docs
+          path: image/rhel/docs
 
       - uses: ./.github/actions/create-concatenated-ui-monorepo-lock
 

Makefile

@@ -567,11 +567,10 @@ main-image: all-builds
 	make docker-build-main-image
 
 $(CURDIR)/image/rhel/bundle.tar.gz:
-	/usr/bin/env DEBUG_BUILD="$(DEBUG_BUILD)" $(CURDIR)/image/rhel/create-bundle.sh $(CURDIR)/image stackrox-data:$(TAG) $(BUILD_IMAGE) $(CURDIR)/image/rhel
+	/usr/bin/env DEBUG_BUILD="$(DEBUG_BUILD)" $(CURDIR)/image/rhel/create-bundle.sh $(CURDIR)/image $(CURDIR)/image/rhel
 
 .PHONY: docker-build-main-image
-docker-build-main-image: copy-binaries-to-image-dir docker-build-data-image central-db-image \
-                         $(CURDIR)/image/rhel/bundle.tar.gz
+docker-build-main-image: copy-binaries-to-image-dir central-db-image $(CURDIR)/image/rhel/bundle.tar.gz
 	docker build \
 		-t stackrox/main:$(TAG) \
 		-t $(DEFAULT_IMAGE_REGISTRY)/main:$(TAG) \
@@ -586,13 +585,6 @@ docker-build-main-image: copy-binaries-to-image-dir docker-build-data-image cent
 	@echo "Built main image for RHEL with tag: $(TAG), image flavor: $(ROX_IMAGE_FLAVOR)"
 	@echo "You may wish to:       export MAIN_IMAGE_TAG=$(TAG)"
 
-.PHONY: docker-build-data-image
-docker-build-data-image:
-	docker build -t stackrox-data:$(TAG) \
-		--label quay.expires-after=$(QUAY_TAG_EXPIRATION) \
-		image/ \
-		--file image/stackrox-data.Dockerfile
-
 .PHONY: docker-build-roxctl-image
 docker-build-roxctl-image:
 	cp -f bin/linux_$(GOARCH)/roxctl image/roxctl/roxctl-linux
@@ -635,7 +627,7 @@ ifdef CI
 else
 	$(SILENT)[ -f image/THIRD_PARTY_NOTICES ] || mkdir -p image/THIRD_PARTY_NOTICES
 endif
-	$(SILENT)[ -d image/docs ] || { echo "Generated docs not found in image/docs. They are required for build."; exit 1; }
+	$(SILENT)[ -d image/rhel/docs ] || { echo "Generated docs not found in image/rhel/docs. They are required for build."; exit 1; }
 
 .PHONY: scale-image
 scale-image: scale-build clean-image
@@ -696,7 +688,7 @@ clean: clean-image
 clean-image:
 	@echo "+ $@"
 	git clean -xf image/bin
-	git clean -xdf image/ui image/docs
+	git clean -xdf image/ui image/rhel/docs
 	git clean -xf integration-tests/mock-grpc-server/image/bin/mock-grpc-server
 	rm -f $(CURDIR)/image/rhel/bundle.tar.gz $(CURDIR)/image/postgres/bundle.tar.gz
 	rm -rf $(CURDIR)/image/rhel/scripts

image/.gitignore

@@ -1,2 +1,3 @@
 /THIRD_PARTY_NOTICES/
 /docs/
+/rhel/docs

image/rhel/Dockerfile

@@ -8,6 +8,15 @@ COPY bundle.tar.gz /
 WORKDIR /bundle
 RUN microdnf install tar gzip && tar -zxf /bundle.tar.gz
 
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as stackrox_data
+
+RUN mkdir /stackrox-data
+RUN microdnf upgrade --nobest && microdnf install openssl zip
+
+WORKDIR /
+COPY fetch-stackrox-data.sh .
+RUN /fetch-stackrox-data.sh
+
 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 
 ARG LABEL_VERSION
@@ -37,12 +46,14 @@ COPY --from=extracted_bundle /bundle/assets/ /assets/
 COPY --from=extracted_bundle /bundle/stackrox/ /stackrox/
 COPY --from=extracted_bundle /bundle/THIRD_PARTY_NOTICES/ /THIRD_PARTY_NOTICES/
 COPY --from=extracted_bundle /bundle/ui/ /ui/
-COPY --from=extracted_bundle /bundle/usr/local/bin/ldb /usr/local/bin/
 COPY --from=extracted_bundle /bundle/snappy.rpm /tmp/
 COPY --from=extracted_bundle /bundle/postgres-libs.rpm /tmp/
 COPY --from=extracted_bundle /bundle/postgres.rpm /tmp/
 COPY --from=extracted_bundle /bundle/go/ /go/
 
+COPY --from=stackrox_data /stackrox-data /stackrox/static-data
+COPY ./docs/api/v1/swagger.json /stackrox/static-data/docs/api/v1/swagger.json
+
 RUN ln -s entrypoint-wrapper.sh /stackrox/admission-control && \
     ln -s entrypoint-wrapper.sh /stackrox/compliance && \
     ln -s entrypoint-wrapper.sh /stackrox/kubernetes-sensor && \

image/rhel/create-bundle.sh

@@ -9,48 +9,18 @@ die() {
     exit 1
 }
 
-image_exists() {
-  if ! docker image inspect "$1" > /dev/null ; then
-     die "Image file $1 not found."
-  fi
-}
-
-extract_from_image() {
-  local image=$1
-  local src=$2
-  local dst=$3
-
-  [[ -n "$image" && -n "$src" && -n "$dst" ]] \
-      || die "extract_from_image: <image> <src> <dst>"
-
-  docker create --name copier "${image}"
-  docker cp "copier:${src}" "${dst}"
-  docker rm copier
-
-  [[ -s $dst ]] || die "file extracted from image is empty: $dst"
-}
-
-INPUT_ROOT="$1"
-DATA_IMAGE="$2"
-BUILDER_IMAGE="$3"
-OUTPUT_DIR="$4"
-# Install the PG repo natively if true (versus using a container)
-NATIVE_PG_INSTALL="${5:-false}"
+INPUT_ROOT="${1:-}"
+OUTPUT_DIR="${2:-}"
 
-[[ -n "$INPUT_ROOT" && -n "$DATA_IMAGE" && -n "$BUILDER_IMAGE" && -n "$OUTPUT_DIR" ]] \
-    || die "Usage: $0 <input-root-directory> <enc-data-image> <builder-image> <output-directory>"
+[[ -n "$INPUT_ROOT" && -n "$OUTPUT_DIR" ]] \
+    || die "Usage: $0 <input-root-directory> <builder-image> <output-directory>"
 [[ -d "$INPUT_ROOT" ]] \
     || die "Input root directory doesn't exist or is not a directory."
 [[ -d "$OUTPUT_DIR" ]] \
     || die "Output directory doesn't exist or is not a directory."
 
 OUTPUT_BUNDLE="${OUTPUT_DIR}/bundle.tar.gz"
 
-# Verify images exist
-if [[ "${DATA_IMAGE}" != "local" ]]; then
-  image_exists "${DATA_IMAGE}"
-fi
-
 # Create tmp directory with stackrox directory structure
 bundle_root="$(mktemp -d)"
 mkdir -p "${bundle_root}"/{assets/downloads/cli,stackrox/bin,ui,usr/local/bin}
@@ -108,16 +78,6 @@ if [[ "$DEBUG_BUILD" == "yes" ]]; then
   fi
 fi
 
-if [[ "${DATA_IMAGE}" != "local" ]]; then
-  # Extract data from data container image
-  mkdir -p "${bundle_root}/stackrox/static-data/"
-  extract_from_image "${DATA_IMAGE}" "/stackrox-data/." "${bundle_root}/stackrox/static-data/"
-  extract_from_image "${BUILDER_IMAGE}" "/usr/local/bin/ldb" "${bundle_root}/usr/local/bin/ldb"
-else
-  cp -a "/stackrox-data" "${bundle_root}/stackrox/static-data/"
-  cp "/usr/local/bin/ldb" "${bundle_root}/usr/local/bin/ldb"
-fi
-
 # Install all the required compression packages for RocksDB to compile
 rpm_base_url="http://mirror.centos.org/centos/8-stream/BaseOS/${arch}/os/Packages"
 rpm_suffix="el8.${arch}.rpm"
@@ -133,21 +93,15 @@ pg_rhel_version="${pg_rhel_major}.${pg_rhel_minor}"
 postgres_url="https://download.postgresql.org/pub/repos/yum/${postgres_major}/redhat/rhel-${pg_rhel_major}-${arch}"
 postgres_repo_url="https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-${arch}/pgdg-redhat-repo-latest.noarch.rpm"
 
-# Determine the Postgres minor version
-if [[ "${NATIVE_PG_INSTALL}" == "true" ]]; then
-    dnf install --disablerepo='*' -y "${postgres_repo_url}"
-    postgres_minor="$(dnf list --disablerepo='*' --enablerepo=pgdg${postgres_major} -y "postgresql${postgres_major}-devel.${arch}" | tail -n 1 | awk '{print $2}').${arch}"
-    echo "PG minor version: ${postgres_minor}"
-else
-    build_dir="$(mktemp -d)"
-    docker build -q -t postgres-minor-image "${build_dir}" -f - <<EOF
+build_dir="$(mktemp -d)"
+docker build -q -t postgres-minor-image "${build_dir}" -f - <<EOF
 FROM registry.access.redhat.com/ubi8/ubi:${pg_rhel_version}
 RUN dnf install --disablerepo='*' -y "${postgres_repo_url}"
-ENTRYPOINT dnf list ${dnf_list_args[@]+"${dnf_list_args[@]}"} --disablerepo='*' --enablerepo=pgdg${postgres_major} -y postgresql${postgres_major}-server.$arch | tail -n 1 | awk '{print \$2}'
+ENTRYPOINT dnf list --disablerepo='*' --enablerepo=pgdg${postgres_major} -y postgresql${postgres_major}-server.$arch | tail -n 1 | awk '{print \$2}'
 EOF
-    postgres_minor="$(docker run --rm postgres-minor-image).${arch}"
-    rm -rf "${build_dir}"
-fi
+
+postgres_minor="$(docker run --rm postgres-minor-image).${arch}"
+rm -rf "${build_dir}"
 
 curl --retry 3 -sS --fail -o "${bundle_root}/postgres.rpm" \
     "${postgres_url}/postgresql${postgres_major}-${postgres_minor}.rpm"

image/rhel/fetch-stackrox-data.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Fetches data used by the stackrox:main image
+
+set -euxo pipefail
+
+fetch_stackrox_data() {
+    mkdir -p /tmp/external-networks
+    local latest_prefix
+    latest_prefix="$(curl --fail https://definitions.stackrox.io/external-networks/latest_prefix)"
+    curl --fail --output /tmp/external-networks/checksum "https://definitions.stackrox.io/${latest_prefix}/checksum"
+    test -s /tmp/external-networks/checksum
+
+    curl --fail --output /tmp/external-networks/networks "https://definitions.stackrox.io/${latest_prefix}/networks"
+    test -s /tmp/external-networks/networks
+
+    sha256sum -c <( echo "$(cat /tmp/external-networks/checksum) /tmp/external-networks/networks" )
+
+    mkdir /stackrox-data/external-networks
+    zip -jr /stackrox-data/external-networks/external-networks.zip /tmp/external-networks
+    rm -rf /tmp/external-networks
+}
+
+fetch_stackrox_data

make/protogen.mk

@@ -15,7 +15,7 @@ API_SERVICE_PROTOS = $(filter api/v1/%, $(SERVICE_PROTOS_REL))
 STORAGE_PROTOS = $(filter storage/%, $(ALL_PROTOS_REL))
 
 GENERATED_BASE_PATH = $(BASE_PATH)/generated
-GENERATED_DOC_PATH = image/docs
+GENERATED_DOC_PATH = image/rhel/docs
 MERGED_API_SWAGGER_SPEC = $(GENERATED_DOC_PATH)/api/v1/swagger.json
 GENERATED_API_DOCS = $(GENERATED_DOC_PATH)/api/v1/reference
 GENERATED_PB_SRCS = $(ALL_PROTOS_REL:%.proto=$(GENERATED_BASE_PATH)/%.pb.go)