diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..86bb3a57 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.sh] +shell_variant = bash +binary_next_line = false +switch_case_indent = true +ij_shell_switch_cases_indented = true +space_redirects = true +keep_padding = false +function_next_line = false \ No newline at end of file diff --git a/.github/workflows/build-all-matrix.yaml b/.github/workflows/build-all-matrix.yaml index 131d97e0..6d34801d 100644 --- a/.github/workflows/build-all-matrix.yaml +++ b/.github/workflows/build-all-matrix.yaml @@ -51,9 +51,8 @@ jobs: id: date_prep run: echo "created=$(date -u +'%Y%m%d-%H%M')" >> "${GITHUB_OUTPUT}" - - name: Run shellcheck # so fail fast in case of bash errors/warnings - id: shellcheck - run: bash build.sh shellcheck + - name: Run lint (shellcheck/shellfmt) # so fail fast in case of bash errors/warnings or unformatted code + run: bash build.sh lint - name: Run the matrix JSON preparation bash script id: prepare-matrix diff --git a/bash/common.sh b/bash/common.sh index 70405541..c2163def 100644 --- a/bash/common.sh +++ b/bash/common.sh @@ -22,23 +22,59 @@ function log() { function install_dependencies() { declare -a debian_pkgs=() - [[ ! -f /usr/bin/jq ]] && debian_pkgs+=("jq") - [[ ! -f /usr/bin/envsubst ]] && debian_pkgs+=("gettext-base") - [[ ! -f /usr/bin/pigz ]] && debian_pkgs+=("pigz") - - # If running on Debian or Ubuntu... - if [[ -f /etc/debian_version ]]; then - # If more than zero entries in the array, install - if [[ ${#debian_pkgs[@]} -gt 0 ]]; then - log warn "Installing dependencies: ${debian_pkgs[*]}" + declare -a brew_pkgs=() + + command -v jq > /dev/null || { + debian_pkgs+=("jq") + brew_pkgs+=("jq") + } + + command -v pigz > /dev/null || { + debian_pkgs+=("pigz") + brew_pkgs+=("pigz") + } + + command -v envsubst > /dev/null || { + debian_pkgs+=("gettext-base") + brew_pkgs+=("gettext") + } + + if [[ "$(uname)" == "Darwin" ]]; then + command -v gtar > /dev/null || brew_pkgs+=("gnu-tar") + command -v greadlink > /dev/null || brew_pkgs+=("coreutils") + command -v gsed > /dev/null || brew_pkgs+=("gnu-sed") + fi + + # If more than zero entries in the array, install + if [[ ${#debian_pkgs[@]} -gt 0 ]]; then + # If running on Debian or Ubuntu... + if [[ -f /etc/debian_version ]]; then + log warn "Installing apt dependencies: ${debian_pkgs[*]}" sudo DEBIAN_FRONTEND=noninteractive apt -o "Dpkg::Use-Pty=0" -y update sudo DEBIAN_FRONTEND=noninteractive apt -o "Dpkg::Use-Pty=0" -y install "${debian_pkgs[@]}" + elif [[ "$(uname)" == "Darwin" ]]; then + log info "Skipping Debian deps installation for Darwin..." + else + log error "Don't know how to install the equivalent of Debian packages *on the host*: ${debian_pkgs[*]} -- teach me!" fi else - log error "Don't know how to install the equivalent of Debian packages: ${debian_pkgs[*]} -- teach me!" + log info "All deps found, no apt installs necessary on host." + fi + + if [[ "$(uname)" == "Darwin" ]]; then + if [[ ${#brew_pkgs[@]} -gt 0 ]]; then + log info "Detected Darwin, assuming 'brew' is available: running 'brew install ${brew_pkgs[*]}'" + brew install "${brew_pkgs[@]}" + fi + + # Re-export PATH with the gnu-version of coreutils, tar, and sed + declare brew_prefix + brew_prefix="$(brew --prefix)" + export PATH="${brew_prefix}/opt/gnu-sed/libexec/gnubin:${brew_prefix}/opt/gnu-tar/libexec/gnubin:${brew_prefix}/opt/coreutils/libexec/gnubin:${PATH}" + log debug "Darwin; PATH is now: ${PATH}" fi - return 0 # there's a shortcircuit above + return 0 } # utility used by inventory.sh to define a kernel/flavour with less-terrible syntax. diff --git a/bash/docker.sh b/bash/docker.sh index f461690f..bca6a85d 100644 --- a/bash/docker.sh +++ b/bash/docker.sh @@ -1,4 +1,18 @@ function check_docker_daemon_for_sanity() { + # LinuxKit is a bit confused about `docker context list` when you're using a non-default context. + # Let's obtain the currect socket from the current context and explicitly export it. + # This allows working on machines with Docker Desktop, colima, and other run-linux-in-a-VM solutions. + declare current_context_docker_socket="" current_docker_context_name="" + current_docker_context_name="$(docker context show)" + current_context_docker_socket="$(docker context inspect "${current_docker_context_name}" | jq -r '.[0].Endpoints.docker.Host')" + log info "Current Docker context ('${current_docker_context_name}') socket: '${current_context_docker_socket}'" + + log debug "Setting DOCKER_HOST to '${current_context_docker_socket}'" + export DOCKER_HOST="${current_context_docker_socket}" + + # Hide Docker, Inc spamming "What's next" et al. + export DOCKER_CLI_HINTS=false + # Shenanigans to go around error control & capture output in the same effort, 'docker info' is slow. declare docker_info docker_buildx_version docker_info="$({ docker info 2> /dev/null && echo "DOCKER_INFO_OK"; } || true)" @@ -23,3 +37,159 @@ function check_docker_daemon_for_sanity() { } } + +# Utility to pull skopeo itself from SKOPEO_IMAGE; checks the local Docker cache and skips if found +function pull_skopeo_image_if_not_in_local_docker_cache() { + # Check if the image is already in the local Docker cache + if docker image inspect "${SKOPEO_IMAGE}" &> /dev/null; then + log info "Skopeo image ${SKOPEO_IMAGE} is already in the local Docker cache; skipping pull." + return 0 + fi + + log info "Pulling Skopeo image ${SKOPEO_IMAGE}..." + + pull_docker_image_from_remote_with_retries "${SKOPEO_IMAGE}" +} + +# Utility to get the most recent tag for a given image, using Skopeo. no retries, a failure is fatal. +# Sets the value of outer-scope variable latest_tag_for_docker_image, so declare it there. +# If extra arguments are present after the image, they are used to grep the tags. +function get_latest_tag_for_docker_image_using_skopeo() { + declare image="$1" + shift + latest_tag_for_docker_image="undetermined_tag" + + # Pull separately to avoid tty hell in the subshell below + pull_skopeo_image_if_not_in_local_docker_cache + + # if extra arguments are present, use them to grep the tags + if [[ -n "$*" ]]; then + latest_tag_for_docker_image="$(docker run "${SKOPEO_IMAGE}" list-tags "docker://${image}" | jq -r ".Tags[]" | grep "${@}" | tail -1)" + else + latest_tag_for_docker_image="$(docker run "${SKOPEO_IMAGE}" list-tags "docker://${image}" | jq -r ".Tags[]" | tail -1)" + fi + log info "Found latest tag: '${latest_tag_for_docker_image}' for image '${image}'" +} + +# Utility to pull from remote, with retries. +function pull_docker_image_from_remote_with_retries() { + declare image="$1" + declare -i retries=3 + declare -i retry_delay=5 + declare -i retry_count=0 + + while [[ ${retry_count} -lt ${retries} ]]; do + if docker pull "${image}"; then + log info "Successfully pulled ${image}" + return 0 + else + log warn "Failed to pull ${image}; retrying in ${retry_delay} seconds..." + sleep "${retry_delay}" + ((retry_count += 1)) + fi + done + + log error "Failed to pull ${image} after ${retries} retries." + exit 1 +} + +# Helper script, for common task of installing packages on a Debian Dockerfile +# always includes curl and downloads ORAS binary too +# takes the relative directory to write the helper to +# sets outer scope dockerfile_helper_filename with the name of the file for the Dockerfile (does not include the directory) +function produce_dockerfile_helper_apt_oras() { + declare target_dir="$1" + declare helper_name="apt-oras-helper.sh" + dockerfile_helper_filename="Dockerfile.autogen.helper.${helper_name}" # this is negated in .dockerignore + + declare fn="${target_dir}${dockerfile_helper_filename}" + cat <<- 'DOWNLOAD_HELPER_SCRIPT' > "${fn}" + #!/bin/bash + set -e + declare oras_version="1.2.2" # See https://github.com/oras-project/oras/releases + # determine the arch to download from current arch + declare oras_arch="unknown" + case "$(uname -m)" in + "x86_64") oras_arch="amd64" ;; + "aarch64" | "arm64") oras_arch="arm64" ;; + *) log error "ERROR: ARCH $(uname -m) not supported by ORAS? check https://github.com/oras-project/oras/releases" && exit 1 ;; + esac + declare oras_down_url="https://github.com/oras-project/oras/releases/download/v${oras_version}/oras_${oras_version}_linux_${oras_arch}.tar.gz" + export DEBIAN_FRONTEND=noninteractive + apt-get -qq -o "Dpkg::Use-Pty=0" update || apt-get -o "Dpkg::Use-Pty=0" update + apt-get -qq install -o "Dpkg::Use-Pty=0" -q -y curl "${@}" || apt-get install -o "Dpkg::Use-Pty=0" -q -y curl "${@}" + curl -sL -o /oras.tar.gz ${oras_down_url} + tar -xvf /oras.tar.gz -C /usr/local/bin/ oras + rm -rf /oras.tar.gz + chmod +x /usr/local/bin/oras + echo -n "ORAS version: " && oras version + DOWNLOAD_HELPER_SCRIPT + log debug "Created apt-oras helper script '${fn}'" +} + +# A huge hack to force the architecture of a Docker image to a specific value. +# This is required for the LinuxKit kernel images: LK expects them to have the correct arch, despite the +# actual contents always being the same. Docker's buildkit tags a locally built image with the host arch. +# Thus change the host arch to the expected arch in the image's manifests via a dump/reimport. +function ensure_docker_image_architecture() { + declare kernel_oci_image="$1" + declare expected_arch="$2" + + # If the host arch is the same as the expected arch, no need to do anything + if [[ "$(uname -m)" == "${expected_arch}" ]]; then + log info "Host architecture is already ${expected_arch}, no need to rewrite Docker image ${kernel_oci_image}" + return 0 + fi + + log info "Rewriting Docker image ${kernel_oci_image} to architecture ${expected_arch}, wait..." + + # Create a temporary directory, use mktemp + declare -g tmpdir + tmpdir="$(mktemp -d)" + log debug "Created temporary directory: ${tmpdir}" + + # Export the image to a tarball + docker save -o "${tmpdir}/original.tar" "${kernel_oci_image}" + + # Untag the hostarch image + docker rmi "${kernel_oci_image}" + + # Create a working dir under the tmpdir + mkdir -p "${tmpdir}/working" + + # Extract the tarball into the working dir + tar -xf "${tmpdir}/original.tar" -C "${tmpdir}/working" + log debug "Extracted tarball to ${tmpdir}/working" + + # Remove the original tarball + rm -f "${tmpdir}/original.tar" + + declare working_blobs_dir="${tmpdir}/working/blobs/sha256" + + # Find all files under working_blobs_dir which are smaller than 2048 bytes + # Use mapfile to create an array of files + declare -a small_files + mapfile -t small_files < <(find "${working_blobs_dir}" -type f -size -2048c) + log debug "Found small blob files: ${small_files[*]}" + + # Replace the architecture in each of the small files + for file in "${small_files[@]}"; do + log debug "Replacing architecture in ${file}" + sed -i "s|\"architecture\":\".....\"|\"architecture\":\"${expected_arch}\"|g" "${file}" # 🤮 + done + + # Create a new tarball with the modified files + tar -cf "${tmpdir}/modified.tar" -C "${tmpdir}/working" . + log debug "Created modified tarball: ${tmpdir}/modified.tar" + + # Remove the working directory + rm -rf "${tmpdir}/working" + + # Import the modified tarball back into the local cache + docker load -i "${tmpdir}/modified.tar" + + # Remove the temporary directory, completely + rm -rf "${tmpdir}" + + log info "Rewrote Docker image ${kernel_oci_image} to architecture ${expected_arch}." +} diff --git a/bash/hook-lk-containers.sh b/bash/hook-lk-containers.sh index c5d812d4..ef42b823 100644 --- a/bash/hook-lk-containers.sh +++ b/bash/hook-lk-containers.sh @@ -66,7 +66,6 @@ function build_hook_linuxkit_container() { return 0 } - function push_hook_linuxkit_container() { declare container_oci_ref="${1}" diff --git a/bash/kernel.sh b/bash/kernel.sh index 0baf30a0..3b24f339 100644 --- a/bash/kernel.sh +++ b/bash/kernel.sh @@ -33,7 +33,7 @@ function kernel_build() { log debug "Kernel build method: ${kernel_info[BUILD_FUNC]}" "${kernel_info[BUILD_FUNC]}" - # Push it to the OCI registry + # Push it to the OCI registry; this discards the os/arch information that BuildKit generates if [[ "${DO_PUSH:-"no"}" == "yes" ]]; then log info "Kernel built; pushing to ${kernel_oci_image}" docker push "${kernel_oci_image}" diff --git a/bash/kernel/kernel_armbian.sh b/bash/kernel/kernel_armbian.sh index 495b1b8e..833fcd1b 100644 --- a/bash/kernel/kernel_armbian.sh +++ b/bash/kernel/kernel_armbian.sh @@ -8,31 +8,13 @@ function calculate_kernel_version_armbian() { declare -g ARMBIAN_KERNEL_BASE_ORAS_REF="${ARMBIAN_BASE_ORAS_REF}/${ARMBIAN_KERNEL_ARTIFACT}" - # If ARMBIAN_KERNEL_VERSION is unset, for using the latest kernel, this requires skopeo & jq + # If ARMBIAN_KERNEL_VERSION is unset, for using the latest kernel if [[ -z "${ARMBIAN_KERNEL_VERSION}" ]]; then log info "ARMBIAN_KERNEL_VERSION is unset, obtaining the most recently pushed-to tag of ${ARMBIAN_KERNEL_BASE_ORAS_REF}" - log info "Getting most recent tag for ${ARMBIAN_KERNEL_BASE_ORAS_REF} via skopeo ${SKOPEO_IMAGE}..." - - # A few tries to pull skopeo. Sorry. quay.io is undergoing an outage. @TODO refactor - declare -i skopeo_pulled=0 skopeo_pull_tries=0 skopeo_max_pull_tries=5 - while [[ "${skopeo_pulled}" -eq 0 && "${skopeo_pull_tries}" -lt "${skopeo_max_pull_tries}" ]]; do - if docker pull "${SKOPEO_IMAGE}"; then - log info "Pulled skopeo image ${SKOPEO_IMAGE} OK" - skopeo_pulled=1 - else - ((skopeo_pull_tries += 1)) - log info "Failed to pull ${SKOPEO_IMAGE}, retrying ${skopeo_pull_tries}/${skopeo_max_pull_tries}" - sleep $((3 + RANDOM % 12)) # sleep a random amount of seconds - fi - done - if [[ "${skopeo_pulled}" -eq 0 ]]; then - log error "Failed to pull after ${skopeo_max_pull_tries} tries, exiting" - exit 1 - fi - - # Pull separately to avoid tty hell in the subshell below - ARMBIAN_KERNEL_VERSION="$(docker run "${SKOPEO_IMAGE}" list-tags "docker://${ARMBIAN_KERNEL_BASE_ORAS_REF}" | jq -r ".Tags[]" | tail -1)" - log info "Using most recent tag: ${ARMBIAN_KERNEL_VERSION}" + declare latest_tag_for_docker_image + get_latest_tag_for_docker_image_using_skopeo "${ARMBIAN_KERNEL_BASE_ORAS_REF}" ".\-S..." # regex to match the tag, like "6.1.84-Sxxxx" + ARMBIAN_KERNEL_VERSION="${latest_tag_for_docker_image}" + log info "Using most recent Armbian kernel tag: ${ARMBIAN_KERNEL_VERSION}" fi # output ID is just the inventory_id @@ -43,31 +25,20 @@ function calculate_kernel_version_armbian() { ARMBIAN_KERNEL_MAJOR_MINOR_POINT="$(echo -n "${ARMBIAN_KERNEL_VERSION}" | cut -d "-" -f 1)" log info "ARMBIAN_KERNEL_MAJOR_MINOR_POINT: ${ARMBIAN_KERNEL_MAJOR_MINOR_POINT}" - declare -g ARMBIAN_KERNEL_DOCKERFILE="kernel/Dockerfile.autogen.armbian.${inventory_id}" - - declare oras_version="1.2.0-rc.1" # @TODO bump this once it's released; yes it's much better than 1.1.x's - # determine the arch to download from current arch - declare oras_arch="unknown" - case "$(uname -m)" in - "x86_64") oras_arch="amd64" ;; - "aarch64" | "arm64") oras_arch="arm64" ;; - *) log error "ERROR: ARCH $(uname -m) not supported by ORAS? check https://github.com/oras-project/oras/releases" && exit 1 ;; - esac - declare oras_down_url="https://github.com/oras-project/oras/releases/download/v${oras_version}/oras_${oras_version}_linux_${oras_arch}.tar.gz" + # A helper script, as escaping bash into a RUN command in Dockerfile is a pain; included in input_hash later + declare dockerfile_helper_filename="undefined.sh" + produce_dockerfile_helper_apt_oras "kernel/" # will create the helper script in kernel/ directory; sets helper_name # Lets create a Dockerfile that will be used to obtain the artifacts needed, using ORAS binary + declare -g ARMBIAN_KERNEL_DOCKERFILE="kernel/Dockerfile.autogen.armbian.${inventory_id}" echo "Creating Dockerfile '${ARMBIAN_KERNEL_DOCKERFILE}'... " cat <<- ARMBIAN_ORAS_DOCKERFILE > "${ARMBIAN_KERNEL_DOCKERFILE}" - FROM debian:stable as downloader - # Install ORAS binary tool from GitHub releases - ENV DEBIAN_FRONTEND=noninteractive - RUN apt -o "Dpkg::Use-Pty=0" update && apt install -o "Dpkg::Use-Pty=0" -y curl dpkg-dev && \ - curl -sL -o /oras.tar.gz ${oras_down_url} && \ - tar -xvf /oras.tar.gz -C /usr/local/bin/ oras && \ - chmod +x /usr/local/bin/oras && \ - oras version + FROM debian:stable AS downloader + # Call the helper to install curl, oras, and dpkg-dev + ADD ./${dockerfile_helper_filename} /apt-oras-helper.sh + RUN bash /apt-oras-helper.sh dpkg-dev - FROM downloader as downloaded + FROM downloader AS downloaded # lets create the output dir WORKDIR /armbian/output @@ -94,11 +65,11 @@ function calculate_kernel_version_armbian() { # Important: this tarball needs to have permissions for the root directory included! Otherwise linuxkit rootfs will have the wrong permissions on / (root) WORKDIR /armbian/modules_only RUN mv /armbian/image/lib /armbian/modules_only/ - RUN echo "Before cleaning: " && du -h -d 10 -x . | sort -h | tail -n 20 + RUN echo "Before cleaning: " && du -h -d 10 -x lib/modules | sort -h | tail -n 20 # Trim the kernel modules to save space; hopefully your required hardware is not included here - RUN rm -rfv ./lib/modules/*/kernel/drivers/net/wireless ./lib/modules/*/kernel/sound ./lib/modules/*/kernel/drivers/media - RUN rm -rfv ./lib/modules/*/kernel/drivers/infiniband - RUN echo "After cleaning: " && du -h -d 10 -x . | sort -h | tail -n 20 + RUN rm -rf ./lib/modules/*/kernel/drivers/net/wireless ./lib/modules/*/kernel/sound ./lib/modules/*/kernel/drivers/media + RUN rm -rf ./lib/modules/*/kernel/drivers/infiniband + RUN echo "After cleaning: " && du -h -d 10 -x lib/modules | sort -h | tail -n 20 RUN tar -cf /armbian/output/kernel.tar . # Create a tarball with the dtbs in usr/lib/linux-image-* @@ -114,8 +85,7 @@ function calculate_kernel_version_armbian() { ARMBIAN_ORAS_DOCKERFILE declare input_hash="" short_input_hash="" - # shellcheck disable=SC2002 # keep cat & hash stdin so we can easily add more factors to the hash one day - input_hash="$(cat "${ARMBIAN_KERNEL_DOCKERFILE}" | sha256sum - | cut -d ' ' -f 1)" + input_hash="$(cat "${ARMBIAN_KERNEL_DOCKERFILE}" "kernel/${dockerfile_helper_filename}" | sha256sum - | cut -d ' ' -f 1)" short_input_hash="${input_hash:0:8}" kernel_oci_version="${ARMBIAN_KERNEL_MAJOR_MINOR_POINT}-${short_input_hash}" armbian_type="${inventory_id#"armbian-"}" # remove the 'armbian-' prefix from inventory_id, but keep the rest. "uefi" has "current/edge" and "arm64/x86" variants. @@ -128,8 +98,10 @@ function build_kernel_armbian() { log info "Building armbian kernel from deb-tar at ${ARMBIAN_KERNEL_FULL_ORAS_REF_DEB_TAR}" log info "Will build Dockerfile ${ARMBIAN_KERNEL_DOCKERFILE}" - # Build the Dockerfile; don't specify platform, our Dockerfile is multiarch, thus you can get build x86 kernels in arm64 hosts and vice-versa + # Don't specify platform, our Dockerfile is multiarch, thus you can build x86 kernels in arm64 hosts and vice-versa ... docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" -t "${kernel_oci_image}" -f "${ARMBIAN_KERNEL_DOCKERFILE}" kernel + # .. but enforce the target arch for LK in the final image via dump/edit-manifests/reimport trick + ensure_docker_image_architecture "${kernel_oci_image}" "${kernel_info['DOCKER_ARCH']}" } function configure_kernel_armbian() { diff --git a/bash/kernel/kernel_default.sh b/bash/kernel/kernel_default.sh index 8c8d7bb2..0c9a7352 100644 --- a/bash/kernel/kernel_default.sh +++ b/bash/kernel/kernel_default.sh @@ -131,9 +131,8 @@ function build_kernel_default() { common_build_args_kernel_default log info "Will build with: ${build_args[*]}" - ( - cd kernel - docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" "${build_args[@]}" -t "${kernel_oci_image}" . - ) - + # Don't specify platform, our Dockerfile is multiarch, thus you can build x86 kernels in arm64 hosts and vice-versa ... + docker buildx build --load "--progress=${DOCKER_BUILDX_PROGRESS_TYPE}" "${build_args[@]}" -t "${kernel_oci_image}" -f kernel/Dockerfile kernel + # .. but enforce the target arch for LK in the final image via dump/edit-manifests/reimport trick + ensure_docker_image_architecture "${kernel_oci_image}" "${kernel_info['DOCKER_ARCH']}" } diff --git a/bash/linuxkit.sh b/bash/linuxkit.sh index aa3836d9..d29f4a1f 100644 --- a/bash/linuxkit.sh +++ b/bash/linuxkit.sh @@ -93,6 +93,11 @@ function linuxkit_build() { declare lk_cache_dir="${CACHE_DIR}/linuxkit" mkdir -p "${lk_cache_dir}" + declare -a lk_debug_args=() + if [[ "${DEBUG}" == "yes" ]]; then + lk_debug_args+=("--verbose" "2") # 0 = quiet, 1 = info, 2 = debug, 3 = trace. + fi + # if LINUXKIT_ISO is set, build an ISO with the kernel and initramfs if [[ -n "${LINUXKIT_ISO}" ]]; then declare lk_iso_output_dir="out" @@ -109,7 +114,7 @@ function linuxkit_build() { ) log info "Building Hook ISO with kernel ${inventory_id} using linuxkit: ${lk_iso_args[*]}" - "${linuxkit_bin}" build "${lk_iso_args[@]}" + "${linuxkit_bin}" build "${lk_debug_args[@]}" "${lk_iso_args[@]}" return 0 fi @@ -124,11 +129,11 @@ function linuxkit_build() { if [[ "${OUTPUT_TARBALL_FILELIST:-"no"}" == "yes" ]]; then log info "OUTPUT_TARBALL_FILELIST=yes; Building Hook (tar/filelist) with kernel ${inventory_id} using linuxkit: ${lk_args[*]}" - "${linuxkit_bin}" build "--format" "tar" "${lk_args[@]}" + "${linuxkit_bin}" build "--format" "tar" "${lk_debug_args[@]}" "${lk_args[@]}" fi log info "Building Hook with kernel ${inventory_id} using linuxkit: ${lk_args[*]}" - "${linuxkit_bin}" build "--format" "kernel+initrd" "${lk_args[@]}" + "${linuxkit_bin}" build "--format" "kernel+initrd" "${lk_debug_args[@]}" "${lk_args[@]}" declare initramfs_path="${lk_output_dir}/hook-initrd.img" # initramfs_path is a gzipped file. obtain the uncompressed byte size, without decompressing it diff --git a/bash/shellcheck.sh b/bash/shellcheck.sh index 9d5cd339..e4703320 100755 --- a/bash/shellcheck.sh +++ b/bash/shellcheck.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash function download_prepare_shellcheck_bin() { - declare SHELLCHECK_VERSION=${SHELLCHECK_VERSION:-0.10.0} # https://github.com/koalaman/shellcheck/releases - log info "Downloading and preparing shellcheck binary for version v${SHELLCHECK_VERSION}..." + declare SHELLCHECK_VERSION=${SHELLCHECK_VERSION:-"0.10.0"} # https://github.com/koalaman/shellcheck/releases + log info "Preparing shellcheck binary for version v${SHELLCHECK_VERSION}..." declare bash_machine="${BASH_VERSINFO[5]}" declare shellcheck_os="" shellcheck_arch="" @@ -49,6 +49,51 @@ function download_prepare_shellcheck_bin() { return 0 } +# Same, but for shellfmt +function download_prepare_shellfmt_bin() { + declare SHELLFMT_VERSION=${SHELLFMT_VERSION:-"3.10.0"} # https://github.com/mvdan/sh/releases/ + log info "Preparing shellfmt binary for version v${SHELLFMT_VERSION}..." + + declare bash_machine="${BASH_VERSINFO[5]}" + declare shellfmt_os="" shellfmt_arch="" + case "$bash_machine" in + *darwin*) shellfmt_os="darwin" ;; + *linux*) shellfmt_os="linux" ;; + *) + log error "unknown os: $bash_machine" + exit 3 + ;; + esac + + case "$bash_machine" in + *aarch64*) shellfmt_arch="arm64" ;; + *x86_64*) shellfmt_arch="amd64" ;; + *) + log error "unknown arch: $bash_machine" + exit 2 + ;; + esac + + declare shellfmt_fn="shfmt_v${SHELLFMT_VERSION}_${shellfmt_os}_${shellfmt_arch}" + declare DOWN_URL="https://github.com/mvdan/sh/releases/download/v${SHELLFMT_VERSION}/${shellfmt_fn}" + declare -g -r SHELLFMT_BIN="${CACHE_DIR}/${shellfmt_fn}" + + if [[ ! -f "${SHELLFMT_BIN}" ]]; then + log info "Cache miss for shellfmt binary, downloading..." + log debug "bash_machine: ${bash_machine}" + log debug "Down URL: ${DOWN_URL}" + log debug "SHELLFMT_BIN: ${SHELLFMT_BIN}" + curl -sL "${DOWN_URL}" -o "${SHELLFMT_BIN}.tmp" + chmod +x "${SHELLFMT_BIN}.tmp" + mv "${SHELLFMT_BIN}.tmp" "${SHELLFMT_BIN}" + fi + + declare -g SHELLFMT_ACTUAL_VERSION="unknown" + SHELLFMT_ACTUAL_VERSION="$("${SHELLFMT_BIN}" --version)" + declare -g -r SHELLFMT_ACTUAL_VERSION="${SHELLFMT_ACTUAL_VERSION}" + log debug "SHELLFMT_ACTUAL_VERSION: ${SHELLFMT_ACTUAL_VERSION}" +} + function run_shellcheck() { declare -a params=() excludes=() @@ -75,3 +120,26 @@ function run_shellcheck() { exit 1 fi } + +function run_shellfmt() { + log info "Running shellfmt ${SHELLFMT_ACTUAL_VERSION} against all bash files, please wait..." + declare -a all_bash_files=() + all_bash_files+=("build.sh") # The root build script + mapfile -t all_bash_files < <(find bash/ -type f -name "*.sh") # All .sh under bash/ + log debug "All bash files: ${all_bash_files[*]}" + + # First, run shellfmt with --diff: it will exit with an error if changes are needed. + if ! "${SHELLFMT_BIN}" --diff "${all_bash_files[@]}"; then + log warn "Shellfmt detected deviations in bash code formatting." + log info "Re-running shellfmt with --write to fix the deviations..." + if ! "${SHELLFMT_BIN}" --write "${all_bash_files[@]}"; then + log error "Shellfmt failed to fix deviations in bash code formatting." + exit 66 + else + log info "Shellfmt fixed deviations in bash code formatting." + fi + exit 1 # So CI breaks + fi + log info "Shellfmt detected no deviations in bash code formatting." + return 0 +} diff --git a/build.sh b/build.sh index d771956d..6f956aca 100755 --- a/build.sh +++ b/build.sh @@ -26,10 +26,10 @@ parse_command_line_arguments "${@}" # which fills the above vars & exports the k declare -g HOOK_KERNEL_OCI_BASE="${HOOK_KERNEL_OCI_BASE:-"quay.io/tinkerbell/hook-kernel"}" declare -g HOOK_LK_CONTAINERS_OCI_BASE="${HOOK_LK_CONTAINERS_OCI_BASE:-"quay.io/tinkerbell/"}" -declare -g SKOPEO_IMAGE="${SKOPEO_IMAGE:-"quay.io/skopeo/stable:latest"}" +declare -g SKOPEO_IMAGE="${SKOPEO_IMAGE:-"quay.io/skopeo/stable:v1.17.0"}" # See https://quay.io/repository/skopeo/stable?tab=tags&tag=latest # See https://github.com/linuxkit/linuxkit/releases -declare -g -r LINUXKIT_VERSION_DEFAULT="1.5.2" # LinuxKit version to use by default; each flavor can set its own too +declare -g -r LINUXKIT_VERSION_DEFAULT="1.5.3" # LinuxKit version to use by default; each flavor can set its own too # Directory to use for storing downloaded artifacts: LinuxKit binary, shellcheck binary, etc. declare -g -r CACHE_DIR="${CACHE_DIR:-"cache"}" @@ -82,6 +82,20 @@ case "${first_param}" in exit 0 ;; + shellfmt) + download_prepare_shellfmt_bin + run_shellfmt # this exits with an error if changes are made + exit 0 + ;; + + lint) + download_prepare_shellcheck_bin + download_prepare_shellfmt_bin + run_shellcheck + run_shellfmt # this exits with an error if changes are made + exit 0 + ;; + gha-matrix) output_gha_matrixes exit 0 diff --git a/images/hook-containerd/Dockerfile b/images/hook-containerd/Dockerfile index adfa92bd..5ccdf423 100644 --- a/images/hook-containerd/Dockerfile +++ b/images/hook-containerd/Dockerfile @@ -1,4 +1,4 @@ -FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as builder +FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e AS builder ARG TARGETPLATFORM @@ -29,7 +29,7 @@ RUN cp bin/containerd bin/ctr bin/containerd-shim bin/containerd-shim-runc-v2 /u RUN strip /usr/bin/containerd /usr/bin/ctr /usr/bin/containerd-shim /usr/bin/containerd-shim-runc-v2 RUN mkdir -p /opt/containerd -FROM scratch as containerd-dev +FROM scratch AS containerd-dev ENTRYPOINT [] WORKDIR / COPY --from=builder /usr/bin/containerd /usr/bin/ctr /usr/bin/containerd-shim /usr/bin/containerd-shim-runc-v2 /usr/bin/ @@ -38,7 +38,7 @@ COPY --from=builder /usr/local/bin/nerdctl /usr/bin/ COPY --from=builder /opt/containerd/ /opt/containerd/ # Dockerfile to build linuxkit/containerd for linuxkit -FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as alpine +FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e AS alpine RUN apk add tzdata binutils RUN mkdir -p /etc/init.d && ln -s /usr/bin/service /etc/init.d/020-containerd @@ -55,4 +55,4 @@ COPY --from=alpine /usr/share/zoneinfo/UTC /etc/localtime COPY --from=alpine /etc/init.d/ /etc/init.d/ COPY etc etc/ COPY --from=alpine /etc/apk /etc/apk/ -COPY --from=alpine /lib/apk /lib/apk/ \ No newline at end of file +COPY --from=alpine /lib/apk /lib/apk/ diff --git a/images/hook-runc/Dockerfile b/images/hook-runc/Dockerfile index a5fb86b8..dfe5bc1c 100644 --- a/images/hook-runc/Dockerfile +++ b/images/hook-runc/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile to build linuxkit/runc for linuxkit -FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as alpine +FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e AS alpine RUN \ apk add \ bash \ @@ -32,4 +32,4 @@ COPY --from=alpine /usr/bin/runc /usr/bin/ COPY --from=alpine /etc/init.d/ /etc/init.d/ COPY --from=alpine /etc/shutdown.d/ /etc/shutdown.d/ COPY --from=alpine /etc/apk /etc/apk/ -COPY --from=alpine /lib/apk /lib/apk/ \ No newline at end of file +COPY --from=alpine /lib/apk /lib/apk/ diff --git a/kernel/.dockerignore b/kernel/.dockerignore index f86da955..a1b4ba04 100644 --- a/kernel/.dockerignore +++ b/kernel/.dockerignore @@ -3,3 +3,4 @@ Dockerfile* Makefile README.md +!Dockerfile.autogen.helper* diff --git a/kernel/Dockerfile b/kernel/Dockerfile index 34903b6f..084e5c89 100644 --- a/kernel/Dockerfile +++ b/kernel/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable as kernel-source-unpacked +FROM debian:stable AS kernel-source-unpacked ENV DEBIAN_FRONTEND=noninteractive # crossbuild-essentials are pretty heavy; here we install for both architecures to maximize Docker layer hit cache rate during development, but only one will be used @@ -30,7 +30,7 @@ RUN set -x && \ cat linux-${KERNEL_VERSION}.tar | tar --absolute-names -x && mv /linux-${KERNEL_VERSION} /linux -FROM kernel-source-unpacked as kernel-with-config +FROM kernel-source-unpacked AS kernel-with-config ARG INPUT_DEFCONFIG ARG KERNEL_ARCH @@ -54,7 +54,7 @@ RUN set -x && make "ARCH=${KERNEL_ARCH}" olddefconfig # docker buildx build --load --progress=plain --build-arg KERNEL_VERSION=5.10.212 --build-arg KERNEL_SERIES=5.10.y -t hook-kernel:builder --target kernel-configurator . # docker run -it -v "$(pwd)":/out-config hook-kernel:builder # Otherwise, since this stage is not referenced anywhere during normal build, it is completely skipped -FROM kernel-with-config as kernel-configurator +FROM kernel-with-config AS kernel-configurator VOLUME /host