batch of fixes and enhancements - early january'2025 (#257)

> further enhancements:
> - can now build (and develop) Hook for any arch under any arch, including for amd64 under Darwin arm64
> - full support for building on macOS+brew, including on arm64
> - added `shellfmt` tool and bash code format enforcing on CI (in addition to shellcheck)
> - avoid pulling skopeo image over and over again

----

### build: implement `shellfmt` (and `lint` which does both shellcheck/fmt)

- for consistent bash formatting
- include an .editorconfig for IDE's

### gha: switch to `lint` (which does both `shellcheck` and `shellfmt`)

### linuxkit: bump 1.5.2 -> 1.5.3

### build: implement build-host dependency handling for macOS+brew

- if on macOS+brew:
  - detects missing deps and installs them with brew
  - exports PATH with brew-based GNU versions first
    - coreutils, gnu-sed and gnu-tar included

### build: Dockerfile: fix FROM xx AS casing

- to squash recent BuildKit warnings

### build: pass `--verbose 2` to linuxkit if `DEBUG=yes`

- can help with some edge cases

### build: refactor skopeo pull and list-tags functions

- this only affects Armbian kernel flavours
- avoids pulling if found in local cache

### build: use skopeo `v1.17.0` instead of latest

- since we now use the local tag

### build: armbian: kernel: refactor Dockerfile with helper

- building the Armbian kernels would produce different hashes depending on the arch of the host
- moving the affected code into the Dockerfile would lead to escaping pain; instead implement a docker.sh helper
- in practice, all code in the Dockerfile is hashed, but the arch decision is now therein and hash won't change
- also, allows for reuse, which is bound to come later

### build: docker: detect & export `DOCKER_HOST` from current `docker context`

- Some Docker-in-VM solutions (like Docker Desktop, colima, etc) set a non-default docker context pointing to the correct socket
- Seems LinuxKit fumbles detecting this and ends up silently failing all local-Docker-daemon cache hits
  - that is fine for CI, where all images are (beforehand) pushed to the registry (and thus LK ends up pulling from remote), but not during local development
  - reported to upstream LinuxKit: linuxkit/linuxkit#4092

### build: kernel: force target arch on cross-built kernel docker image manifest

- our kernel builds are done in arch-independent Dockerfiles
- but those get the build-host's architecture, despite the contents being correct
- when locally developing on a kernel that is != host-arch
  - those get the host-arch in the image
  - but LinuxKit refuses to use it due to arch mismatch
- (when pushed to a registry, the arch info is discarded, and LK is ok with that)
- thus
  - introduce `ensure_docker_image_architecture(imagetag, arch)`
    - which just hacks at manifests via a docker save/docker load
  - call it from both default and armbian kernel builds

### build: docker: avoid Docker Inc's "What's next" hints

- enough spam already, thanks

-----

Signed-off-by: Ricardo Pardini <[email protected]>

.editorconfig

@@ -0,0 +1,16 @@
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.sh]
+shell_variant = bash
+binary_next_line = false
+switch_case_indent = true
+ij_shell_switch_cases_indented = true
+space_redirects = true
+keep_padding = false
+function_next_line = false

.github/workflows/build-all-matrix.yaml

@@ -51,9 +51,8 @@ jobs:
         id: date_prep
         run: echo "created=$(date -u +'%Y%m%d-%H%M')" >> "${GITHUB_OUTPUT}"
 
-      - name: Run shellcheck # so fail fast in case of bash errors/warnings
-        id: shellcheck
-        run: bash build.sh shellcheck
+      - name: Run lint (shellcheck/shellfmt) # so fail fast in case of bash errors/warnings or unformatted code
+        run: bash build.sh lint
 
       - name: Run the matrix JSON preparation bash script
         id: prepare-matrix

bash/common.sh

@@ -22,23 +22,59 @@ function log() {
 
 function install_dependencies() {
 	declare -a debian_pkgs=()
-	[[ ! -f /usr/bin/jq ]] && debian_pkgs+=("jq")
-	[[ ! -f /usr/bin/envsubst ]] && debian_pkgs+=("gettext-base")
-	[[ ! -f /usr/bin/pigz ]] && debian_pkgs+=("pigz")
-
-	# If running on Debian or Ubuntu...
-	if [[ -f /etc/debian_version ]]; then
-		# If more than zero entries in the array, install
-		if [[ ${#debian_pkgs[@]} -gt 0 ]]; then
-			log warn "Installing dependencies: ${debian_pkgs[*]}"
+	declare -a brew_pkgs=()
+
+	command -v jq > /dev/null || {
+		debian_pkgs+=("jq")
+		brew_pkgs+=("jq")
+	}
+
+	command -v pigz > /dev/null || {
+		debian_pkgs+=("pigz")
+		brew_pkgs+=("pigz")
+	}
+
+	command -v envsubst > /dev/null || {
+		debian_pkgs+=("gettext-base")
+		brew_pkgs+=("gettext")
+	}
+
+	if [[ "$(uname)" == "Darwin" ]]; then
+		command -v gtar > /dev/null || brew_pkgs+=("gnu-tar")
+		command -v greadlink > /dev/null || brew_pkgs+=("coreutils")
+		command -v gsed > /dev/null || brew_pkgs+=("gnu-sed")
+	fi
+
+	# If more than zero entries in the array, install
+	if [[ ${#debian_pkgs[@]} -gt 0 ]]; then
+		# If running on Debian or Ubuntu...
+		if [[ -f /etc/debian_version ]]; then
+			log warn "Installing apt dependencies: ${debian_pkgs[*]}"
 			sudo DEBIAN_FRONTEND=noninteractive apt -o "Dpkg::Use-Pty=0" -y update
 			sudo DEBIAN_FRONTEND=noninteractive apt -o "Dpkg::Use-Pty=0" -y install "${debian_pkgs[@]}"
+		elif [[ "$(uname)" == "Darwin" ]]; then
+			log info "Skipping Debian deps installation for Darwin..."
+		else
+			log error "Don't know how to install the equivalent of Debian packages *on the host*: ${debian_pkgs[*]} -- teach me!"
 		fi
 	else
-		log error "Don't know how to install the equivalent of Debian packages: ${debian_pkgs[*]} -- teach me!"
+		log info "All deps found, no apt installs necessary on host."
+	fi
+
+	if [[ "$(uname)" == "Darwin" ]]; then
+		if [[ ${#brew_pkgs[@]} -gt 0 ]]; then
+			log info "Detected Darwin, assuming 'brew' is available: running 'brew install ${brew_pkgs[*]}'"
+			brew install "${brew_pkgs[@]}"
+		fi
+
+		# Re-export PATH with the gnu-version of coreutils, tar, and sed
+		declare brew_prefix
+		brew_prefix="$(brew --prefix)"
+		export PATH="${brew_prefix}/opt/gnu-sed/libexec/gnubin:${brew_prefix}/opt/gnu-tar/libexec/gnubin:${brew_prefix}/opt/coreutils/libexec/gnubin:${PATH}"
+		log debug "Darwin; PATH is now: ${PATH}"
 	fi
 
-	return 0 # there's a shortcircuit above
+	return 0
 }
 
 # utility used by inventory.sh to define a kernel/flavour with less-terrible syntax.

bash/docker.sh

@@ -1,4 +1,18 @@
 function check_docker_daemon_for_sanity() {
+	# LinuxKit is a bit confused about `docker context list` when you're using a non-default context.
+	# Let's obtain the currect socket from the current context and explicitly export it.
+	# This allows working on machines with Docker Desktop, colima, and other run-linux-in-a-VM solutions.
+	declare current_context_docker_socket="" current_docker_context_name=""
+	current_docker_context_name="$(docker context show)"
+	current_context_docker_socket="$(docker context inspect "${current_docker_context_name}" | jq -r '.[0].Endpoints.docker.Host')"
+	log info "Current Docker context ('${current_docker_context_name}') socket: '${current_context_docker_socket}'"
+
+	log debug "Setting DOCKER_HOST to '${current_context_docker_socket}'"
+	export DOCKER_HOST="${current_context_docker_socket}"
+
+	# Hide Docker, Inc spamming "What's next" et al.
+	export DOCKER_CLI_HINTS=false
+
 	# Shenanigans to go around error control & capture output in the same effort, 'docker info' is slow.
 	declare docker_info docker_buildx_version
 	docker_info="$({ docker info 2> /dev/null && echo "DOCKER_INFO_OK"; } || true)"
@@ -23,3 +37,159 @@ function check_docker_daemon_for_sanity() {
 	}
 
 }
+
+# Utility to pull skopeo itself from SKOPEO_IMAGE; checks the local Docker cache and skips if found
+function pull_skopeo_image_if_not_in_local_docker_cache() {
+	# Check if the image is already in the local Docker cache
+	if docker image inspect "${SKOPEO_IMAGE}" &> /dev/null; then
+		log info "Skopeo image ${SKOPEO_IMAGE} is already in the local Docker cache; skipping pull."
+		return 0
+	fi
+
+	log info "Pulling Skopeo image ${SKOPEO_IMAGE}..."
+
+	pull_docker_image_from_remote_with_retries "${SKOPEO_IMAGE}"
+}
+
+# Utility to get the most recent tag for a given image, using Skopeo. no retries, a failure is fatal.
+# Sets the value of outer-scope variable latest_tag_for_docker_image, so declare it there.
+# If extra arguments are present after the image, they are used to grep the tags.
+function get_latest_tag_for_docker_image_using_skopeo() {
+	declare image="$1"
+	shift
+	latest_tag_for_docker_image="undetermined_tag"
+
+	# Pull separately to avoid tty hell in the subshell below
+	pull_skopeo_image_if_not_in_local_docker_cache
+
+	# if extra arguments are present, use them to grep the tags
+	if [[ -n "$*" ]]; then
+		latest_tag_for_docker_image="$(docker run "${SKOPEO_IMAGE}" list-tags "docker://${image}" | jq -r ".Tags[]" | grep "${@}" | tail -1)"
+	else
+		latest_tag_for_docker_image="$(docker run "${SKOPEO_IMAGE}" list-tags "docker://${image}" | jq -r ".Tags[]" | tail -1)"
+	fi
+	log info "Found latest tag: '${latest_tag_for_docker_image}' for image '${image}'"
+}
+
+# Utility to pull from remote, with retries.
+function pull_docker_image_from_remote_with_retries() {
+	declare image="$1"
+	declare -i retries=3
+	declare -i retry_delay=5
+	declare -i retry_count=0
+
+	while [[ ${retry_count} -lt ${retries} ]]; do
+		if docker pull "${image}"; then
+			log info "Successfully pulled ${image}"
+			return 0
+		else
+			log warn "Failed to pull ${image}; retrying in ${retry_delay} seconds..."
+			sleep "${retry_delay}"
+			((retry_count += 1))
+		fi
+	done
+
+	log error "Failed to pull ${image} after ${retries} retries."
+	exit 1
+}
+
+# Helper script, for common task of installing packages on a Debian Dockerfile
+# always includes curl and downloads ORAS binary too
+# takes the relative directory to write the helper to
+# sets outer scope dockerfile_helper_filename with the name of the file for the Dockerfile (does not include the directory)
+function produce_dockerfile_helper_apt_oras() {
+	declare target_dir="$1"
+	declare helper_name="apt-oras-helper.sh"
+	dockerfile_helper_filename="Dockerfile.autogen.helper.${helper_name}" # this is negated in .dockerignore
+
+	declare fn="${target_dir}${dockerfile_helper_filename}"
+	cat <<- 'DOWNLOAD_HELPER_SCRIPT' > "${fn}"
+		#!/bin/bash
+		set -e
+		declare oras_version="1.2.2" # See https://github.com/oras-project/oras/releases
+		# determine the arch to download from current arch
+		declare oras_arch="unknown"
+		case "$(uname -m)" in
+			"x86_64") oras_arch="amd64" ;;
+			"aarch64" | "arm64") oras_arch="arm64" ;;
+			*) log error "ERROR: ARCH $(uname -m) not supported by ORAS? check https://github.com/oras-project/oras/releases" && exit 1 ;;
+		esac
+		declare oras_down_url="https://github.com/oras-project/oras/releases/download/v${oras_version}/oras_${oras_version}_linux_${oras_arch}.tar.gz"
+		export DEBIAN_FRONTEND=noninteractive
+		apt-get -qq -o "Dpkg::Use-Pty=0" update || apt-get -o "Dpkg::Use-Pty=0" update
+		apt-get -qq install -o "Dpkg::Use-Pty=0" -q -y curl "${@}" || apt-get install -o "Dpkg::Use-Pty=0" -q -y curl "${@}"
+		curl -sL -o /oras.tar.gz ${oras_down_url}
+		tar -xvf /oras.tar.gz -C /usr/local/bin/ oras
+		rm -rf /oras.tar.gz
+		chmod +x /usr/local/bin/oras
+		echo -n "ORAS version: " && oras version
+	DOWNLOAD_HELPER_SCRIPT
+	log debug "Created apt-oras helper script '${fn}'"
+}
+
+# A huge hack to force the architecture of a Docker image to a specific value.
+# This is required for the LinuxKit kernel images: LK expects them to have the correct arch, despite the
+#  actual contents always being the same. Docker's buildkit tags a locally built image with the host arch.
+# Thus change the host arch to the expected arch in the image's manifests via a dump/reimport.
+function ensure_docker_image_architecture() {
+	declare kernel_oci_image="$1"
+	declare expected_arch="$2"
+
+	# If the host arch is the same as the expected arch, no need to do anything
+	if [[ "$(uname -m)" == "${expected_arch}" ]]; then
+		log info "Host architecture is already ${expected_arch}, no need to rewrite Docker image ${kernel_oci_image}"
+		return 0
+	fi
+
+	log info "Rewriting Docker image ${kernel_oci_image} to architecture ${expected_arch}, wait..."
+
+	# Create a temporary directory, use mktemp
+	declare -g tmpdir
+	tmpdir="$(mktemp -d)"
+	log debug "Created temporary directory: ${tmpdir}"
+
+	# Export the image to a tarball
+	docker save -o "${tmpdir}/original.tar" "${kernel_oci_image}"
+
+	# Untag the hostarch image
+	docker rmi "${kernel_oci_image}"
+
+	# Create a working dir under the tmpdir
+	mkdir -p "${tmpdir}/working"
+
+	# Extract the tarball into the working dir
+	tar -xf "${tmpdir}/original.tar" -C "${tmpdir}/working"
+	log debug "Extracted tarball to ${tmpdir}/working"
+
+	# Remove the original tarball
+	rm -f "${tmpdir}/original.tar"
+
+	declare working_blobs_dir="${tmpdir}/working/blobs/sha256"
+
+	# Find all files under working_blobs_dir which are smaller than 2048 bytes
+	# Use mapfile to create an array of files
+	declare -a small_files
+	mapfile -t small_files < <(find "${working_blobs_dir}" -type f -size -2048c)
+	log debug "Found small blob files: ${small_files[*]}"
+
+	# Replace the architecture in each of the small files
+	for file in "${small_files[@]}"; do
+		log debug "Replacing architecture in ${file}"
+		sed -i "s|\"architecture\":\".....\"|\"architecture\":\"${expected_arch}\"|g" "${file}" # 🤮
+	done
+
+	# Create a new tarball with the modified files
+	tar -cf "${tmpdir}/modified.tar" -C "${tmpdir}/working" .
+	log debug "Created modified tarball: ${tmpdir}/modified.tar"
+
+	# Remove the working directory
+	rm -rf "${tmpdir}/working"
+
+	# Import the modified tarball back into the local cache
+	docker load -i "${tmpdir}/modified.tar"
+
+	# Remove the temporary directory, completely
+	rm -rf "${tmpdir}"
+
+	log info "Rewrote Docker image ${kernel_oci_image} to architecture ${expected_arch}."
+}

bash/hook-lk-containers.sh

@@ -66,7 +66,6 @@ function build_hook_linuxkit_container() {
 	return 0
 }
 
-
 function push_hook_linuxkit_container() {
 	declare container_oci_ref="${1}"
 

bash/kernel.sh

@@ -33,7 +33,7 @@ function kernel_build() {
 	log debug "Kernel build method: ${kernel_info[BUILD_FUNC]}"
 	"${kernel_info[BUILD_FUNC]}"
 
-	# Push it to the OCI registry
+	# Push it to the OCI registry; this discards the os/arch information that BuildKit generates
 	if [[ "${DO_PUSH:-"no"}" == "yes" ]]; then
 		log info "Kernel built; pushing to ${kernel_oci_image}"
 		docker push "${kernel_oci_image}"