From 4de4bbe470ebf1d7049926d50f87a655fbfbadb2 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Wed, 17 Jul 2024 14:50:10 +0100 Subject: [PATCH 1/6] Drop unused create-manifest script Signed-off-by: James Le Cuirot --- create-manifest | 108 --------------------------------- lib/common.sh | 5 -- manifest-template.xml.envsubst | 37 ----------- 3 files changed, 150 deletions(-) delete mode 100755 create-manifest delete mode 100644 manifest-template.xml.envsubst diff --git a/create-manifest b/create-manifest deleted file mode 100755 index 0a510a8..0000000 --- a/create-manifest +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" - -if [ "$VERSION" = "" ] || [ "$SDK_VERSION" = "" ] || [ "$CHANNEL" = "" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - echo "$0:" - echo "This script will create a branch in the manifest repository checked out in $SCRIPTFOLDER/../manifest and push the branch and tag." - echo "The repository is cloned from github.com/kinvolk/manifest.git if it does not exist (origin in an existing repository must point there" - echo "or pushing fails or does some unwanted action)." - echo "Set VERSION, SDK_VERSION, and CHANNEL as environment variables, e.g., VERSION=2345.3.0 SDK_VERSION=2345.2.0 CHANNEL=stable $0" - echo - echo "It creates tags only for the manifest repository. Tags for the repositories scripts, coreos-overlay, and portage-stable should be" - echo "created first with ./tag-release (which can be rerun on build failures to introduce fixes without changing the manifest)." - echo "By default the manifest references refs/tags/CHANNEL-VERSION for the repositories scripts, coreos-overlay, and portage-stable" - echo "and simply refs/heads/master for the others because their revisions are defined by the CROS_WORKON_COMMIT in the respective ebuild files." - echo "Set the environment variables SCRIPTS_REF, OVERLAY_REF, PORTAGE_REF, or DEFAULT_REF to specify any other reference." - exit 1 -fi - -set -euo pipefail - -SCRIPTS_REF="${SCRIPTS_REF-refs/tags/$CHANNEL-$VERSION}" -OVERLAY_REF="${OVERLAY_REF-refs/tags/$CHANNEL-$VERSION}" -PORTAGE_REF="${PORTAGE_REF-refs/tags/$CHANNEL-$VERSION}" -DEFAULT_REF="${DEFAULT_REF-refs/heads/master}" - -echo "Running with CHANNEL=$CHANNEL VERSION=$VERSION SDK_VERSION=$SDK_VERSION" -echo "SCRIPTS_REF=$SCRIPTS_REF OVERLAY_REF=$OVERLAY_REF PORTAGE_REF=$PORTAGE_REF" -echo "DEFAULT_REF=$DEFAULT_REF" - -cd "$SCRIPTFOLDER/.." - -MANIFESTFOLDER="manifest" - -if [ -d "${MANIFESTFOLDER}" ]; then - git -C "${MANIFESTFOLDER}" fetch --prune origin -else - git clone git@github.com:kinvolk/manifest.git "${MANIFESTFOLDER}" -fi - -cd "${MANIFESTFOLDER}" - -if [ "$(git status --porcelain || echo failed)" != "" ]; then - echo "Error: Unexpected output of git status:" - git status - exit 1 -fi - -BRANCHNAME="flatcar-$CHANNEL-$VERSION" -EXISTS=0 -echo "Preparing branch" -git checkout flatcar-master -git pull -git branch "$BRANCHNAME" || EXISTS=1 -git checkout "$BRANCHNAME" -if [ "$EXISTS" = 1 ]; then - echo "Warning: Reusing existing branch $BRANCHNAME, will try to pull." - git pull || echo "Warning: Pulling failed. Ignore the above output if the branch just exists locally." -fi -echo "Preparing files" -sed -E -i "s/(FLATCAR_VERSION=)(.*)/\1$VERSION/" version.txt -sed -E -i "s/(FLATCAR_VERSION_ID=)(.*)/\1$VERSION/" version.txt -sed -E -i "s/(FLATCAR_SDK_VERSION=)(.*)/\1$SDK_VERSION/" version.txt - -echo "Removing old build-*.xml files" -git rm ./build-*.xml || echo "Warning: Could not delete old files" - -FILENAME="build-$(echo "$VERSION" | cut -d '.' -f 1).xml" - -export SCRIPTS_REF OVERLAY_REF PORTAGE_REF DEFAULT_REF -cat "$SCRIPTFOLDER/manifest-template.xml.envsubst" | envsubst '$SCRIPTS_REF $OVERLAY_REF $PORTAGE_REF $DEFAULT_REF' > "$FILENAME" -# Note: appc-acbuild, appc-spec, rkt, and systemd always stay at refs/heads/master because they do not have flatcar-master or build branches - -ln -fs "$FILENAME" default.xml -cp "$FILENAME" release.xml - -MAJOR="${VERSION%%.*}" - -MAINT="flatcar" -if [ "$CHANNEL" = lts ] && [ "$MAJOR" = "2605" ]; then - MAINT="flatcar-lts" -fi - -echo "Creating maintenance.xml for $MAINT-$MAJOR branches" - -SCRIPTS_REF="refs/heads/$MAINT-$MAJOR" -OVERLAY_REF="refs/heads/$MAINT-$MAJOR" -PORTAGE_REF="refs/heads/$MAINT-$MAJOR" - -export SCRIPTS_REF OVERLAY_REF PORTAGE_REF DEFAULT_REF -cat "$SCRIPTFOLDER/manifest-template.xml.envsubst" | envsubst '$SCRIPTS_REF $OVERLAY_REF $PORTAGE_REF $DEFAULT_REF' > maintenance.xml - -echo "Adding changed files" -git add "$FILENAME" release.xml default.xml version.txt maintenance.xml -echo "Committing manifest" -git commit -m "build $VERSION" --author 'Flatcar Buildbot ' -echo "Pushing branch" - -git push --force-with-lease --set-upstream origin "$BRANCHNAME" -TAG="v$VERSION" -echo "Deleting tag $TAG if it exists" -git tag -d "$TAG" || echo "No local tags deleted" -git push --delete origin "$TAG" || echo "No remote tags deleted" -echo "Tagging commit" -git tag -s "$TAG" -m "$TAG" -echo "Pushing tag" -git push origin "$TAG" -echo "Done" diff --git a/lib/common.sh b/lib/common.sh index 8d98848..445f4f3 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -36,8 +36,3 @@ export FLATCAR_REPOS=( "update_engine" "updateservicectl" ) - -export FLATCAR_REPOS_MANIFEST=( - "manifest" - "manifest-builds" -) diff --git a/manifest-template.xml.envsubst b/manifest-template.xml.envsubst deleted file mode 100644 index 6a75a86..0000000 --- a/manifest-template.xml.envsubst +++ /dev/null @@ -1,37 +0,0 @@ - - - Your sources have been sync'd successfully. Note that only the revisions for the coreos-overlay, scripts, and portage-stable repositories are relevant. Revisions for the other repositories are specified in the ebuild files. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 655fcd2332cd13a40fbc65eebfffb9cd6cf77ed8 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 18 Jul 2024 18:06:54 +0100 Subject: [PATCH 2/6] Drop obsolete compare-flatcar-with-gentoo script A newer version of this script lives in the reports repository. Signed-off-by: James Le Cuirot --- compare-flatcar-with-gentoo | 809 ------------------------------------ 1 file changed, 809 deletions(-) delete mode 100755 compare-flatcar-with-gentoo diff --git a/compare-flatcar-with-gentoo b/compare-flatcar-with-gentoo deleted file mode 100755 index 85cbd1f..0000000 --- a/compare-flatcar-with-gentoo +++ /dev/null @@ -1,809 +0,0 @@ -#!/bin/bash - -# This script tries to compare the package versions in portage-stable, -# coreos-overlay and gentoo. When done, prints some statistics to -# standard output. -# -# Example invocations: -# ==================== -# -# ./compare-flatcar-with-gentoo -# -# export COREOS_OVERLAY=../../co-repo -# export PORTAGE_STABLE=../../ps-repo -# export GENTOO=../../g-repo -# export JSON=x -# ./compare-flatcar-with-gentoo -# -# Environment variables: -# ====================== -# -# COREOS_OVERLAY: -# -# A path to the coreos-overlay git repository. -# -# PORTAGE_STABLE: -# -# A path to the portage-stable git repository. -# -# GENTOO: -# -# A path to the gentoo git repository. -# -# JSON: -# -# If not empty, print statistics in JSON format. -# -# Environment variables used for debugging: -# ========================================= -# -# WORKDIR: -# -# A path to a working directory that will hold temporary files. -# -# KEEP_WORKDIR: -# -# If specified to non-empty value, then the workdir will be printed to -# standard error if verbose output is enabled and will not be removed -# after script finishes. -# -# VERBOSE: -# -# Print debugging stuff to standard error. - -set -euo pipefail - -shopt -s extglob - -this_dir="$(dirname "${0}")" - -: ${COREOS_OVERLAY:="${this_dir}/../coreos-overlay"} -: ${PORTAGE_STABLE:="${this_dir}/../portage-stable"} -: ${GENTOO:="${this_dir}/../gentoo"} -: ${JSON:=} -: ${WORKDIR:="$(mktemp --directory "${this_dir}/cfwg.XXXXXXXXXX")"} -: ${KEEP_WORKDIR:=} -: ${VERBOSE:=} - -stderr() { - printf '%s\n' "$*" >/dev/stderr -} - -debug() { - if [[ -z "${VERBOSE}" ]]; then - return - fi - stderr "$@" -} - -warn() { - stderr 'WARNING:' "$@" -} - -drop_trailing_slash() { - local var_name="${1}" - - local -n var_ref="${var_name}" - - var_ref="${var_ref%%*(/)}" -} - -drop_trailing_slash PORTAGE_STABLE -drop_trailing_slash COREOS_OVERLAY -drop_trailing_slash GENTOO -drop_trailing_slash WORKDIR - -mkdir -p "${WORKDIR}" - -if [[ -z "${KEEP_WORKDIR}" ]]; then - trap "rm -rf '${WORKDIR}'" EXIT -else - debug "WORKDIR='${WORKDIR}'" -fi - -debug "PORTAGE_STABLE=${PORTAGE_STABLE}" -debug "COREOS_OVERLAY=${COREOS_OVERLAY}" -debug "GENTOO=${GENTOO}" - -prepend_with() { - local field_count="${1}" - local item="${2}" - local arr_name="${3}" - local -n arr_ref="${arr_name}" - - while [[ "${#arr_ref[@]}" -lt "${field_count}" ]]; do - arr_ref=("${item}" "${arr_ref[@]}") - done -} - -append_with() { - local field_count="${1}" - local item="${2}" - local arr_name="${3}" - local -n arr_ref="${arr_name}" - - while [[ "${#arr_ref[@]}" -lt "${field_count}" ]]; do - arr_ref+=("${item}") - done -} - -strip_leading_zeros() { - local num_name="${1}" - - local -n num_ref="${num_name}" - - # strip all leading zeros, will work only if extglob is enabled - num_ref="${num_ref##*(0)}" - - if [[ -z "${num_ref}" ]]; then - num_ref='0' - fi -} - -get_plain_version() { - local ver="${1}" - local arr_name="${2}" - - # strip everything after first underline or dash, whichever comes - # first, so for strings like 1.2.3_alpha or 1.2.3-r2 we get 1.2.3 - ver="${ver%%[-_]*}" - IFS='.' read -ra "${arr_name}" <<< "${ver}" - - local i - local -n arr_ref="${arr_name}" - - for i in $(seq 0 "$((${#arr_ref[@]}-1))"); do - strip_leading_zeros "${arr_name}[${i}]" - done -} - -# last version field may have a letter -split_last_version_field() { - local field="${1}" - local num_name="${2}" - local word_name="${3}" - - local -n num_ref="${num_name}" - local -n word_ref="${word_name}" - - num_ref="${field%%*([a-z])}" - word_ref="${field##*([0-9])}" - - strip_leading_zeros "${num_name}" -} - -get_suffixes() { - local ver="${1}" - local arr_name="${2}" - local ver_tmp - - if [[ ! "${ver}" =~ _ ]]; then - local -n arr_ref="${arr_name}" - arr_ref=() - return - fi - ver_tmp="${ver#*_}" - ver="${ver_tmp%-*}" - IFS='_' read -ra "${arr_name}" <<< "${ver}" -} - -split_suffix() { - local suffix="${1}" - local word_name="${2}" - local num_name="${3}" - - local -n word_ref="${word_name}" - local -n num_ref="${num_name}" - - word_ref="${suffix%%*([0-9])}" - num_ref="${suffix##*([a-z])}" - - strip_leading_zeros "${num_name}" -} - -get_revision() { - local version="${1}" - local rev_name="${2}" - - local -n rev_ref="${rev_name}" - - if [[ ! "${version}" =~ '-' ]]; then - rev_ref='0' - return - fi - rev_ref="${version##*-r}" - - strip_leading_zeros "${rev_name}" -} - -declare -A SUFFIXES=( - ['alpha']='0' - ['beta']='1' - ['pre']=2 - ['rc']=3 - # empty string can't be a key - # ['']=4 - ['p']=5 -) - -get_suffix_word_index() { - local word="${1}" - local index_ref="${2}" - - local -n index="${index_ref}" - - # we can't put an empty key into the hash map - if [[ -z "${word}" ]]; then - index=4 - return - fi - # assign -1 to invalid suffix - index="${SUFFIXES[${word}]--1}" -} - -gentoo_vercmp() { - local a_ver="${1}" - local b_ver="${2}" - local res_name="${3}" - - local -n res_ref="${res_name}" - - local -a a_nums - local -a b_nums - local a_last_field_idx - local b_last_field_idx - - get_plain_version "${a_ver}" a_nums - get_plain_version "${b_ver}" b_nums - a_last_field_idx="$((${#a_nums[@]}-1))" - b_last_field_idx="$((${#b_nums[@]}-1))" - - if [[ "${#a_nums[@]}" -ne "${#b_nums[@]}" ]]; then - if [[ "${#a_nums[@]}" -gt "${#b_nums[@]}" ]]; then - append_with "${#a_nums[@]}" 0 b_nums - else - append_with "${#b_nums[@]}" 0 a_nums - fi - fi - - local i - local a - local b - local a_num - local a_word - local b_num - local b_word - - for i in $(seq 0 $(("${#a_nums[@]}" - 1))); do - a="${a_nums[${i}]}" - b="${b_nums[${i}]}" - - if [[ "${i}" -eq "${a_last_field_idx}" ]]; then - split_last_version_field "${a}" a_num a_word - else - a_num="${a}" - a_word='' - if [[ ! "${a_num}" =~ [0-9] ]]; then - warn "Invalid version field '${a_num}' in '${a_ver}', only last version field can contain letters." - fi - fi - if [[ "${i}" -eq "${b_last_field_idx}" ]]; then - split_last_version_field "${b}" b_num b_word - else - b_num="${b}" - b_word='' - if [[ ! "${b_num}" =~ [0-9] ]]; then - warn "Invalid version field '${b_num}' in '${b_ver}', only last version field can contain letters." - fi - fi - - if [[ "${a_num}" -gt "${b_num}" ]]; then - res_ref=">v${i}n" - return - fi - if [[ "${a_num}" -lt "${b_num}" ]]; then - res_ref=" "${b_word}" ]]; then - res_ref=">v${i}w" - return - fi - if [[ "${a_word}" < "${b_word}" ]]; then - res_ref=""${WORKDIR}/${name}-pkgs" - git -C "${repo}" log -1 --pretty='%H' >"${WORKDIR}/${name}-hash" - git -C "${repo}" log -1 --pretty='%cD' >"${WORKDIR}/${name}-date" - fi - done - - cat "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/portage-stable-pkgs" | sort -u >"${WORKDIR}/flatcar-pkgs" - fi - - debug "finding all duplicated packages in portage-stable and coreos-overlay" - comm -1 -2 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/portage-stable-pkgs" >"${WORKDIR}/duplicated-pkgs" - - debug "finding all flatcar-only packages in portage-stable" - comm -2 -3 "${WORKDIR}/portage-stable-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/portage-stable-flatcar-only-pkgs" - debug "finding all flatcar-only packages in coreos-overlay" - comm -2 -3 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/coreos-overlay-flatcar-only-pkgs" - debug "finding all flatcar-only packages" - cat "${WORKDIR}/portage-stable-flatcar-only-pkgs" "${WORKDIR}/coreos-overlay-flatcar-only-pkgs" | sort -u >"${WORKDIR}/flatcar-only-pkgs" - - debug "finding all common packages in portage-stable" - comm -1 -2 "${WORKDIR}/portage-stable-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/portage-stable-common-pkgs" - debug "finding all common packages in coreos-overlay" - comm -1 -2 "${WORKDIR}/coreos-overlay-pkgs" "${WORKDIR}/gentoo-pkgs" >"${WORKDIR}/coreos-overlay-common-pkgs" - debug "finding all common packages" - cat "${WORKDIR}/coreos-overlay-common-pkgs" "${WORKDIR}/portage-stable-common-pkgs" | sort -u >"${WORKDIR}/common-pkgs" -fi - -get_pkg_name() { - local name="${1}" - echo -n "${name#*/}" -} - -get_max_version() { - local pkg="${1}" - local max_version_name="${2}" - - shift 2 - - local repo - local pkg_path_prefix - local -a ebuilds - local e - local version - local result - local -n max_version_ref="${max_version_name}" - - # this is quite a low version - max_version_ref='0_alpha_alpha_alpha' - - for repo; do - if [[ ! -d "${repo}/${pkg}" ]]; then - continue - fi - - pkg_path_prefix="${repo}/${pkg}/$(get_pkg_name "${pkg}")-" - ebuilds=( "${pkg_path_prefix}"*.ebuild ) - - for e in "${ebuilds[@]}"; do - # strip prefix - version="$(cut_leading_path "${pkg_path_prefix}" <<<"${e}")" - # strip .ebuild - version="${version:0:-7}" - # ignore versions like 9999 or 99999999 - if [[ -z "${version//9}" ]]; then - continue - fi - result='=' - gentoo_vercmp "${version}" "${max_version_ref}" result - if [[ "${result}" = '>'* ]]; then - max_version_ref="${version}" - fi - done - return - done -} - -find_max_versions() { - local input="${1}" - local output="${2}" - - # create the file in case of an empty output - touch "${output}" - - while read -r line; do - debug " - ${line}" - gentoo_max='' - get_max_version "${line}" gentoo_max "${GENTOO}" - flatcar_max='' - get_max_version "${line}" flatcar_max "${PORTAGE_STABLE}" "${COREOS_OVERLAY}" - result='=' - gentoo_vercmp "${gentoo_max}" "${flatcar_max}" result - printf '%s %s\n' "${line}" "gentoo:${gentoo_max} flatcar:${flatcar_max} result:${result}" >>"${output}" - done < "${input}" -} - -if [[ ! -e "${WORKDIR}/portage-stable-common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages in portage-stable" - find_max_versions "${WORKDIR}/portage-stable-common-pkgs" "${WORKDIR}/portage-stable-common-pkgs-max-versions" -fi - -if [[ ! -e "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages in coreos-overlay" - find_max_versions "${WORKDIR}/coreos-overlay-common-pkgs" "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" -fi - -if [[ ! -e "${WORKDIR}/common-pkgs-max-versions" ]]; then - debug "finding max versions of common packages" - cat "${WORKDIR}/portage-stable-common-pkgs-max-versions" "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | sort -u >"${WORKDIR}/common-pkgs-max-versions" -fi - -debug "done" - -percent() { - local num="${1}" - local den="${2}" - - bc <<< "scale=2; (${num} * 100) / ${den}" -} - -duplicates="$(cat "${WORKDIR}/duplicated-pkgs" | wc -l)" - -if [[ "${duplicates}" -gt 0 ]]; then - warn 'There are duplicated packages in portage-stable and coreos-overlay.' - warn 'Some stats may be skewed or make no sense.' - warn 'BEGIN DUPLICATES' - cat "${WORKDIR}/duplicated-pkgs" >/dev/stderr - warn 'END DUPLICATES' -fi - -# non-failing grep -nfgrep() { - grep "$@" || : -} - -ps_hash="$(cat "${WORKDIR}/portage-stable-hash")" -ps_date="$(cat "${WORKDIR}/portage-stable-date")" -co_hash="$(cat "${WORKDIR}/coreos-overlay-hash")" -co_date="$(cat "${WORKDIR}/coreos-overlay-date")" -gentoo_hash="$(cat "${WORKDIR}/gentoo-hash")" -gentoo_date="$(cat "${WORKDIR}/gentoo-date")" - -total="$(cat "${WORKDIR}/flatcar-pkgs" | wc -l)" -ps_pkgs="$(cat "${WORKDIR}/portage-stable-pkgs" | wc -l)" -co_pkgs="$(cat "${WORKDIR}/coreos-overlay-pkgs" | wc -l)" -ours="$(cat "${WORKDIR}/flatcar-only-pkgs" | wc -l)" -total_common="$(cat "${WORKDIR}/common-pkgs" | wc -l)" -ps_common="$(cat "${WORKDIR}/portage-stable-common-pkgs" | wc -l)" -co_common="$(cat "${WORKDIR}/coreos-overlay-common-pkgs" | wc -l)" -newer="$(nfgrep -e 'result:<' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -equal="$(nfgrep -e 'result:=' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated="$(nfgrep -e 'result:>' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" -outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/common-pkgs-max-versions" | wc -l)" - -ps_newer="$(nfgrep -e 'result:<' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_equal="$(nfgrep -e 'result:=' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated="$(nfgrep -e 'result:>' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" -ps_outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/portage-stable-common-pkgs-max-versions" | wc -l)" - -co_newer="$(nfgrep -e 'result:<' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_equal="$(nfgrep -e 'result:=' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated="$(nfgrep -e 'result:>' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_rev="$(nfgrep -e 'result:>r' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_suf="$(nfgrep -e 'result:>s' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" -co_outdated_ver="$(nfgrep -e 'result:>v' "${WORKDIR}/coreos-overlay-common-pkgs-max-versions" | wc -l)" - -out_type='spaces' -if [[ -n "${JSON}" ]]; then - out_type='json' -fi - -out_init() { - "${out_type}_out_init" "${@}" -} - -out_start() { - "${out_type}_out_start" "${@}" -} - -out_kv() { - "${out_type}_out_kv" "${@}" -} - -out_end() { - "${out_type}_out_end" "${@}" -} - -out_fini() { - "${out_type}_out_fini" "${@}" -} - -spaces_indent=' ' -spaces_first_len=40 -spaces_second_len=8 -spaces_first_group_start=1 - -spaces_out_init() { - : -} - -spaces_out_start() { - if [[ ${spaces_first_group_start} -eq 0 ]]; then - printf '\n'; - else - spaces_first_group_start=0 - fi - printf '%s:\n' "$*" -} - -spaces_repeat() { - local var_name="${1}" - local count="${2}" - local -n var_ref="${var_name}" - local str="$(printf '=%.0s' $(seq 1 "${count}"))" - - var_ref="${str//=/ }" -} - -spaces_out_kv() { - local k="${1}" - local t="${2}" - local v="${3}" - local kl="${#k}" - local ktabl=$((spaces_first_len-kl-1)) - local ktab - - spaces_repeat ktab "${ktabl}" - - if [[ "${#}" -gt 3 ]]; then - local vl="${#v}" - local vtabl=$((spaces_second_len-vl)) - local vtab - local p="${4}" - - spaces_repeat vtab "${vtabl}" - - printf '%s%s:%s%s%s(%s%%)\n' "${spaces_indent}" "${k}" "${ktab}" "${v}" "${vtab}" "${p}" - else - printf '%s%s:%s%s\n' "${spaces_indent}" "${k}" "${ktab}" "${v}" - fi -} - -spaces_out_end() { - : -} - -spaces_out_fini() { - : -} - -declare -a json_out_array - -json_drop_comma() { - local l="${#json_out_array[@]}" - - if [[ "${l}" -eq 0 ]]; then - return - fi - - local i=$((l - 1)) - - json_out_array[${i}]="${json_out_array[${i}]%,}" -} - -json_out_init() { - json_out_array=('{') -} - -json_out_start() { - json_out_array+=(" \"${*}\": {") -} - -json_escape_string() { - local var_name="${1}" - local str="${2}" - local -n var_ref="${var_name}" - # escape backslashes - local escaped="${str//\\/\\\\}" - # escape double quotes - var_ref="\"${escaped//\"/\\\"}\"" -} - -json_v() { - local var_name="${1}" - local t="${2}" - local v="${3}" - local -n var_ref="${var_name}" - - case "${t}" in - i|f) - var_ref="${v}" - ;; - *) - json_escape_string "${var_name}" "${v}" - ;; - esac -} - -json_out_kv() { - local k="${1}" - local t="${2}" - local v="${3}" - local pv - - json_v pv "${t}" "${v}" - json_out_array+=(" \"${k}\": ${pv},") -} - -json_out_end() { - json_drop_comma - json_out_array+=(' },') -} - -json_out_fini() { - json_drop_comma - json_out_array+=('}') - printf '%s\n' "${json_out_array[@]}" -} - -out_init - -out_start 'meta' -out_kv 'portage-stable hash' s "${ps_hash}" -out_kv 'portage-stable date' s "${ps_date}" -out_kv 'coreos-overlay hash' s "${co_hash}" -out_kv 'coreos-overlay date' s "${co_date}" -out_kv 'gentoo hash' s "${gentoo_hash}" -out_kv 'gentoo date' s "${gentoo_date}" -out_end - -out_start 'general' -out_kv 'total packages' i "${total}" -out_kv 'portage-stable' i "${ps_pkgs}" "$(percent "${ps_pkgs}" "${total}")" -out_kv 'coreos-overlay' i "${co_pkgs}" "$(percent "${co_pkgs}" "${total}")" -out_kv 'our packages' i "${ours}" "$(percent "${ours}" "${total}")" -out_kv 'common packages' i "${total_common}" "$(percent "${total_common}" "${total}")" -out_kv 'common packages in portage-stable' i "${ps_common}" "$(percent "${ps_common}" "${total}")" -out_kv 'common packages in coreos-overlay' i "${co_common}" "$(percent "${co_common}" "${total}")" -out_kv 'newer packages' i "${newer}" "$(percent "${newer}" "${total}")" -out_kv 'equal packages' i "${equal}" "$(percent "${equal}" "${total}")" -out_kv 'outdated packages' i "${outdated}" "$(percent "${outdated}" "${total}")" -out_kv 'outdated revision packages' i "${outdated_rev}" "$(percent "${outdated_rev}" "${total}")" -out_kv 'outdated suffix packages' i "${outdated_suf}" "$(percent "${outdated_suf}" "${total}")" -out_kv 'outdated version packages' i "${outdated_ver}" "$(percent "${outdated_ver}" "${total}")" -out_end - -out_start 'portage-stable' -out_kv 'total packages' i "${ps_pkgs}" -out_kv 'common packages' i "${ps_common}" "$(percent "${ps_common}" "${ps_pkgs}")" -out_kv 'newer packages' i "${ps_newer}" "$(percent "${ps_newer}" "${ps_pkgs}")" -out_kv 'equal packages' i "${ps_equal}" "$(percent "${ps_equal}" "${ps_pkgs}")" -out_kv 'all outdated packages' i "${ps_outdated}" "$(percent "${ps_outdated}" "${ps_pkgs}")" -out_kv 'outdated revision packages' i "${ps_outdated_rev}" "$(percent "${ps_outdated_rev}" "${ps_pkgs}")" -out_kv 'outdated suffix packages' i "${ps_outdated_suf}" "$(percent "${ps_outdated_suf}" "${ps_pkgs}")" -out_kv 'outdated version packages' i "${ps_outdated_ver}" "$(percent "${ps_outdated_ver}" "${ps_pkgs}")" -out_end - -out_start 'coreos-overlay' -out_kv 'total packages' i "${co_pkgs}" -out_kv 'common packages' i "${co_common}" "$(percent "${co_common}" "${co_pkgs}")" -out_kv 'newer packages' i "${co_newer}" "$(percent "${co_newer}" "${co_pkgs}")" -out_kv 'equal packages' i "${co_equal}" "$(percent "${co_equal}" "${co_pkgs}")" -out_kv 'all outdated packages' i "${co_outdated}" "$(percent "${co_outdated}" "${co_pkgs}")" -out_kv 'outdated revision packages' i "${co_outdated_rev}" "$(percent "${co_outdated_rev}" "${co_pkgs}")" -out_kv 'outdated suffix packages' i "${co_outdated_suf}" "$(percent "${co_outdated_suf}" "${co_pkgs}")" -out_kv 'outdated version packages' i "${co_outdated_ver}" "$(percent "${co_outdated_ver}" "${co_pkgs}")" -out_end - -out_fini From 1905e264f92a082a6900fa2b83058810bcbc412f Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Wed, 17 Jul 2024 15:01:52 +0100 Subject: [PATCH 3/6] Update coreos-overlay-diff.py following transition to flatcar_workon CROS_WORKON_COMMIT has been replaced by EGIT_COMMIT in ebuilds. Signed-off-by: James Le Cuirot --- coreos-overlay-diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coreos-overlay-diff.py b/coreos-overlay-diff.py index 35aeecb..6696f69 100755 --- a/coreos-overlay-diff.py +++ b/coreos-overlay-diff.py @@ -9,7 +9,7 @@ from pathlib import Path parser = argparse.ArgumentParser(description="Compare two coreos-overlay branches including " - "dereferened CROS_WORKON_COMMIT branches of " + "dereferened EGIT_COMMIT branches of " "repositories located in coreos-overlay/../.") parser.add_argument("THEIRS", type=str, help="Reference/branch to compare to") @@ -141,7 +141,7 @@ def display_difference(from_theirs, to_ours, name, recurse=False): if repo in repo_map: repo = repo_map[repo] # @TODO: Add DOCKER_GITCOMMIT, COMMIT_ID, CONTAINERD_COMMIT - if "CROS_WORKON_COMMIT=" in line: + if "EGIT_COMMIT=" in line: if repo == "": raise Exception("No repo seen for: " + line) is_theirs = line.startswith("-") From 92e2e66402f25c80513fd9783139fbbd0144a030 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Wed, 17 Jul 2024 15:26:24 +0100 Subject: [PATCH 4/6] Remove submodule handling now that no release channel uses them portage-stable and coreos-overlay used to be submodules of scripts, but this is no longer true in any release channel, so we can drop any code that handles the migration. Submodule-related arguments are still passed to git where they make sense just in case we do start using submodules that don't need special handling. Some of these scripts seem a little overly complex now that they only handle the scripts repo, but let's not rewrite them now. Signed-off-by: James Le Cuirot --- devstats | 2 +- lib/common.sh | 2 -- mirror-repos-branch | 8 +------ show-changes | 58 +++++++-------------------------------------- tag-release | 58 ++++++++++++--------------------------------- 5 files changed, 25 insertions(+), 103 deletions(-) diff --git a/devstats b/devstats index c50eb1e..7c3ed4e 100755 --- a/devstats +++ b/devstats @@ -32,7 +32,7 @@ fi # Excluded: flatcar/ignition flatcar/afterburn flatcar/ign-converter flatcar/grub # (because they are upstream projects) -REPOS=(flatcar/scripts flatcar/coreos-overlay flatcar/portage-stable flatcar/flatcar-website +REPOS=(flatcar/scripts flatcar/flatcar-website flatcar/init flatcar/flatcar-linux-update-operator flatcar/flatcar-build-scripts flatcar/mantle flatcar/update-ssh-keys flatcar/container-linux-config-transpiler flatcar/locksmith flatcar/bootengine flatcar/baselayout diff --git a/lib/common.sh b/lib/common.sh index 445f4f3..fd2b5ba 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -9,7 +9,6 @@ export FLATCAR_REPOS=( "bootengine" "chromite" "coreos-cloudinit" - "coreos-overlay" "dev-util" "docker" "efunctions" @@ -23,7 +22,6 @@ export FLATCAR_REPOS=( "mantle" "mayday" "nss-altfiles" - "portage-stable" "rkt" "scripts" "sdnotify-proxy" diff --git a/mirror-repos-branch b/mirror-repos-branch index a315319..36e67f6 100755 --- a/mirror-repos-branch +++ b/mirror-repos-branch @@ -30,7 +30,7 @@ fi . ./lib/common.sh if [ "${ALL_REPOS}" = "0" ]; then - FLATCAR_REPOS=("coreos-overlay" "portage-stable" "scripts") + FLATCAR_REPOS=("scripts") fi REPOS_DIR=$(mktemp -d "${PWD}/.mirror-repos.XXXXXXXXXX") @@ -41,12 +41,6 @@ trap '{ export EXT="$?"; rm -rf "${REPOS_DIR}" && exit "${EXT}"; }' EXIT cd "${REPOS_DIR}" -# Check if we are in the monorepo case -[ ! -d scripts ] && git clone --recurse-submodules "git@github.com:flatcar/scripts" -if [ "$(git -C scripts show "origin/${SRC_BUILD_BRANCH}":.gitmodules 2>/dev/null)" = "" ]; then - FLATCAR_REPOS=("scripts") -fi - for repo in "${FLATCAR_REPOS[@]}"; do # Note: systemd repo has neither flatcar-build-* branches nor flatcar-master # branch. So we should skip systemd completely. diff --git a/show-changes b/show-changes index 49ead14..29d6873 100755 --- a/show-changes +++ b/show-changes @@ -7,14 +7,12 @@ SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "Usage: $0 OLD [NEW]" echo "Shows the changes between the git references by assembling the changelog/ folder entries" - echo "By default the script assumes that it is being run in the folder that contains the coreos-overlay, portage-stable, and scripts repository folders. Set SCRIPTS_REPO, COREOS_OVERLAY_REPO and PORTAGE_REPO to contain the paths to the respective repos if the default assumption is wrong." + echo "By default the script assumes that it is being run in the folder that contains the scripts repository folder. Set SCRIPTS_REPO to contain the path to the repo if the default assumption is wrong." echo "The NEW reference can be omitted and will then default to HEAD." exit 1 fi : "${SCRIPTS_REPO:=scripts}" -: "${COREOS_OVERLAY_REPO:=coreos-overlay}" -: "${PORTAGE_STABLE_REPO:=portage-stable}" OLD="$1" NEW="${2-HEAD}" @@ -32,7 +30,7 @@ fi echo "_Changes since **${OLD_FMT}**_" if [[ ${FETCH} = 1 ]]; then - for repo in coreos-overlay portage-stable scripts; do + for repo in scripts; do var_name=${repo//-/_} var_name="${var_name^^}_REPO" if [[ -d ${!var_name} ]]; then @@ -61,44 +59,15 @@ for section in security bugfixes changes updates; do exit 1 esac echo - for repo in coreos-overlay portage-stable scripts; do + for repo in scripts; do var_name=${repo//-/_} var_name="${var_name^^}_REPO" OLDREF="${OLD}" NEWREF="${NEW}" - OLDREPOPATH="${!var_name}" - NEWREPOPATH="${!var_name}" - OLDPREPEND="" - NEWPREPEND="" - if [ "${repo}" != "scripts" ]; then - # Check if we are in the monorepo case where we can use the scripts ref or not - if [ "$(git -C "${SCRIPTS_REPO}" show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then - # Old version is not a monorepo but has submodules. - # Find the pinned submodule refs because there may be no release tags inside the submodules - # Pipe to awk instead of using --object-only for git 2.35 support - OLDREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - # We can't assume anymore that the submodule repo is available under scripts/ - if [ ! -d "${OLDREPOPATH}" ]; then - git clone "git@github.com:flatcar/${repo}.git" "${OLDREPOPATH}" - fi - else - OLDPREPEND="sdk_container/src/third_party/${repo}/" - OLDREPOPATH="${SCRIPTS_REPO}" - fi - if [ "$(git -C "${SCRIPTS_REPO}" show "${NEW}":.gitmodules 2>/dev/null)" != "" ]; then - # New version is not a monorepo but has submodules. - NEWREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - if [ ! -d "${NEWREPOPATH}" ]; then - git clone "git@github.com:flatcar/${repo}.git" "${NEWREPOPATH}" - fi - else - NEWPREPEND="sdk_container/src/third_party/${repo}/" - NEWREPOPATH="${SCRIPTS_REPO}" - fi - fi - if [ "${section}" = "security" ] && [ "${repo}" = "coreos-overlay" ]; then - FROM_KERNEL=$(git -C "${OLDREPOPATH}" show "${OLDREF}":"${OLDPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) - TO_KERNEL=$(git -C "${NEWREPOPATH}" show "${NEWREF}":"${NEWPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + REPOPATH="${!var_name}" + if [ "${section}" = "security" ] && [ "${repo}" = "scripts" ]; then + FROM_KERNEL=$(git -C "${REPOPATH}" show "${OLDREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + TO_KERNEL=$(git -C "${REPOPATH}" show "${NEWREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) if [ "${FROM_KERNEL}" != "${TO_KERNEL}" ]; then KERNEL_ENTRIES=$("${SCRIPTFOLDER}"/show-fixed-kernel-cves.py -f "${FROM_KERNEL}" -t "${TO_KERNEL}") if [ "${KERNEL_ENTRIES}" != "" ]; then @@ -107,18 +76,7 @@ for section in security bugfixes changes updates; do fi fi - # The assumption is that the old ref is really older, so we can assume that old would have submodules while new doesn't have them anymore - if [ "${OLDREPOPATH}" != "${NEWREPOPATH}" ] && [ "${NEWREPOPATH}" = "${SCRIPTS_REPO}" ]; then - # One patch before the ./checkout helper disappeared we still had submodules - LAST_SUBMOD_SCRIPTS_REF="$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- checkout)~1" - LAST_SUBMOD_REF=$(git -C "${SCRIPTS_REPO}" ls-tree "${LAST_SUBMOD_SCRIPTS_REF}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - # The patch that removed the submodule overrides README has the merge history - FIRST_MONO_REF=$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- sdk_container/git-override/README.md) - git -C "${OLDREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${LAST_SUBMOD_REF}" -- "${OLDPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${FIRST_MONO_REF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - else - git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } - fi + git -C "${REPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } # The -x 'sh -c "cat \"$REMOTE\"" --' command assumes that new changes have their own changelog files, # and thus ignores the LOCAL file (which is the empty /dev/null) and prints out the REMOTE completly. # If an existing file got changed, we assume that this is just a correction for the old change but diff --git a/tag-release b/tag-release index 3d2296e..1fb469a 100755 --- a/tag-release +++ b/tag-release @@ -4,16 +4,15 @@ SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" if [ "${SDK_VERSION}" = "" ] || [ "$VERSION" = "" ] || [ "$CHANNEL" = "" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "$0:" - echo "This script will create and push release tags in the repositories scripts, coreos-overlay, and portage-stable" - echo "checked out in $SCRIPTFOLDER/../(scripts|coreos-overlay|portage-stable). The repositories will be cloned from" - echo "github.com/flatcar/(scripts|coreos-overlay|portage-stable).git if they do not exist (origin in an existing" - echo "repository must point there or pushing fails or does some unwanted action)." + echo "This script will create and push release tags in the scripts repository checked out in" + echo "$SCRIPTFOLDER/../scripts. The repository will be cloned from github.com/flatcar/scripts.git" + echo "if it does not exist (origin in an existing repository must point there or pushing fails or does some unwanted action)." echo "Set VERSION, SDK_VERSION, and CHANNEL as environment variables, e.g., VERSION=2345.3.0 SDK_VERSION=2345.0.0 CHANNEL=stable $0" echo echo "It creates tags in the form CHANNEL-VERSION for each of the three repositories. By default it fetches origin and" echo "creates a tag from origin/flatcar-MAJOR for each of the three repositories, with MAJOR being the first part of" echo "VERSION when splitting by dots." - echo "Set the environment variables SCRIPTS_REF, OVERLAY_REF, PORTAGE_REF to specify a specific commit for the tag." + echo "Set the environment variable SCRIPTS_REF to specify a specific commit for the tag." exit 1 fi @@ -21,64 +20,42 @@ set -euo pipefail MAJOR="${VERSION%%.*}" -MAINT="flatcar" -if [ "$CHANNEL" = lts ] && [ "$MAJOR" = "2605" ]; then - MAINT="flatcar-lts" -fi - -SCRIPTS_REF="${SCRIPTS_REF-origin/$MAINT-$MAJOR}" -OVERLAY_REF="${OVERLAY_REF-origin/$MAINT-$MAJOR}" -PORTAGE_REF="${PORTAGE_REF-origin/$MAINT-$MAJOR}" +SCRIPTS_REF="${SCRIPTS_REF-origin/flatcar-$MAJOR}" echo "Running with CHANNEL=$CHANNEL VERSION=$VERSION MAJOR=$MAJOR" -echo "SCRIPTS_REF=$SCRIPTS_REF OVERLAY_REF=$OVERLAY_REF PORTAGE_REF=$PORTAGE_REF" - -REPOS="coreos-overlay portage-stable scripts" -# scripts should be last as it binds the other two together as submodules, if they exist - -# Check if we are in the monorepo, in this case only work on "scripts" -cd "$SCRIPTFOLDER/.." -if [ ! -d "scripts" ]; then - git clone --recurse-submodules 'git@github.com:flatcar/scripts.git' -fi -git -C scripts fetch origin -if [ "$(git -C scripts show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" = "" ]; then - REPOS="scripts" -fi +echo "SCRIPTS_REF=$SCRIPTS_REF" +REPOS="scripts" for REPO in ${REPOS}; do echo "Preparing ${REPO}" cd "$SCRIPTFOLDER/.." if [ ! -d "${REPO}" ]; then - git clone --recurse-submodules "git@github.com:flatcar/${REPO}.git" + git clone "git@github.com:flatcar/${REPO}.git" fi cd "${REPO}" git fetch origin TAG="${CHANNEL}-${VERSION}" [ "${REPO}" = "scripts" ] && REF="${SCRIPTS_REF}" - [ "${REPO}" = "coreos-overlay" ] && REF="${OVERLAY_REF}" - [ "${REPO}" = "portage-stable" ] && REF="${PORTAGE_REF}" echo "Deleting tag ${TAG} if it exists in ${REPO}" git tag -d "$TAG" || echo "No local tags deleted" git push --delete origin "$TAG" || echo "No remote tags deleted" - # Check if we have to update the submodules while tagging if [ "${REPO}" = "scripts" ]; then - if [ "${REF}" != "origin/$MAINT-$MAJOR" ]; then - echo "Error: can't find the scripts branch to push the updated submodule to, you can't overwrite SCRIPTS_REF anymore" + if [ "${REF}" != "origin/flatcar-$MAJOR" ]; then + echo "Error: can't find the scripts branch, you can't overwrite SCRIPTS_REF anymore" exit 1 fi - echo "Checking out scripts branch $MAINT-$MAJOR to update submodules and create version file" - git checkout --force --recurse-submodules -B "$MAINT-$MAJOR" "origin/$MAINT-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; } + echo "Checking out scripts branch flatcar-$MAJOR to create version file" + git checkout --force -B "flatcar-$MAJOR" "origin/flatcar-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; } if [ "$(git status --porcelain --untracked-files=no)" != "" ]; then echo "Error: uncommitted changes found which will be lost" exit 1 fi - git pull --recurse-submodules || { echo "Error: could not pull the branch in your 'scripts' repo" ; exit 1 ; } - if [ "$(git log HEAD.."origin/$MAINT-$MAJOR")" != "" ] || ! git diff --quiet "origin/$MAINT-$MAJOR" ; then + git pull || { echo "Error: could not pull the branch in your 'scripts' repo" ; exit 1 ; } + if [ "$(git log HEAD.."origin/flatcar-$MAJOR")" != "" ] || ! git diff --quiet "origin/flatcar-$MAJOR" ; then echo "Error: local changes in your 'scripts' repo" exit 1 fi @@ -86,13 +63,8 @@ for REPO in ${REPOS}; do ( source sdk_lib/sdk_container_common.sh source ci-automation/ci_automation_common.sh - if [ "$(git show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" != "" ]; then - cd "sdk_container/src/third_party/coreos-overlay"; git fetch --all --tags --force; cd - - cd "sdk_container/src/third_party/portage-stable"; git fetch --all --tags --force; cd - - update_submodules "${OVERLAY_REF}" "${PORTAGE_REF}" - fi create_versionfile "${SDK_VERSION}" "${VERSION}" - SIGN=1 update_and_push_version "${TAG}" "$MAINT-$MAJOR" + SIGN=1 update_and_push_version "${TAG}" "flatcar-$MAJOR" ) else # Tag the other repos From f17f3fc1e86525674d1452a2d958dbcc3f706d96 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 18 Jul 2024 18:03:29 +0100 Subject: [PATCH 5/6] Adjust scripts following move and rename of package repositories Compatibility symlinks have been added to the scripts repo, but they are not needed by these scripts. `git show` cannot follow them, so we need to explicit check both the new and old locations. coreos-overlay-diff.py has been renamed to flatcar-overlay-diff.py and now accepts --flatcar-overlay, but --coreos-overlay has been kept as an alias. Signed-off-by: James Le Cuirot --- coreos-overlay-diff.py => flatcar-overlay-diff.py | 12 ++++++------ show-changes | 4 ++-- sync-with-gentoo | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename coreos-overlay-diff.py => flatcar-overlay-diff.py (95%) diff --git a/coreos-overlay-diff.py b/flatcar-overlay-diff.py similarity index 95% rename from coreos-overlay-diff.py rename to flatcar-overlay-diff.py index 6696f69..c025a53 100755 --- a/coreos-overlay-diff.py +++ b/flatcar-overlay-diff.py @@ -8,13 +8,13 @@ import os from pathlib import Path -parser = argparse.ArgumentParser(description="Compare two coreos-overlay branches including " +parser = argparse.ArgumentParser(description="Compare two flatcar-overlay branches including " "dereferened EGIT_COMMIT branches of " - "repositories located in coreos-overlay/../.") + "repositories located in flatcar-overlay/../.") parser.add_argument("THEIRS", type=str, help="Reference/branch to compare to") parser.add_argument("--ours", type=str, help="Our reference (defaults to \"HEAD\")") -parser.add_argument("--coreos-overlay", type=str, help="Path to coreos-overlay repository (defaults to \".\")") +parser.add_argument("--flatcar-overlay", "--coreos-overlay", type=str, help="Path to flatcar-overlay repository (defaults to \".\")") parser.add_argument("--no-color", dest="no_color", action="store_true", help="Don't pipe diff through colordiff") parser.add_argument("--no-commits", dest="no_commits", action="store_true", help="Don't log which commits are missing") parser.add_argument("--no-diffs", dest="no_diffs", action="store_true", help="Don't show diffs") @@ -22,10 +22,10 @@ "(ignoring cherry picks without a merge commit as cherry-pick-for can do)") parser.add_argument("--diff-style", choices=["standard", "word-diff", "icdiff", "colordiff", "delta"], help="Instead of standard git diff, use either git --word-diff, git icdiff, or pipe the existing diff through colordiff or delta") -parser.set_defaults(ours="HEAD", coreos_overlay=".", no_color=False, no_commits=False, no_diffs=False) +parser.set_defaults(ours="HEAD", flatcar_overlay=".", no_color=False, no_commits=False, no_diffs=False) args = parser.parse_args() -base_folder = str(Path(args.coreos_overlay + "/../").resolve()) + "/" +base_folder = str(Path(args.flatcar_overlay + "/../").resolve()) + "/" if args.diff_style == "colordiff": if not which("colordiff"): @@ -186,7 +186,7 @@ def display_difference(from_theirs, to_ours, name, recurse=False): ours = "" -os.chdir(args.coreos_overlay) +os.chdir(args.flatcar_overlay) display_difference(args.THEIRS, args.ours, os.path.basename(os.path.abspath(".")), recurse=True) if warnings: print("Encountered some errors when trying to compare recursively, probably due to deleted files:") diff --git a/show-changes b/show-changes index 29d6873..b18f66a 100755 --- a/show-changes +++ b/show-changes @@ -66,8 +66,8 @@ for section in security bugfixes changes updates; do NEWREF="${NEW}" REPOPATH="${!var_name}" if [ "${section}" = "security" ] && [ "${repo}" = "scripts" ]; then - FROM_KERNEL=$(git -C "${REPOPATH}" show "${OLDREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) - TO_KERNEL=$(git -C "${REPOPATH}" show "${NEWREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + FROM_KERNEL=$({ git -C "${REPOPATH}" show "${OLDREF}":repos/flatcar-overlay/sys-kernel/coreos-kernel/ 2>/dev/null || git -C "${REPOPATH}" show "${OLDREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/; } | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) + TO_KERNEL=$({ git -C "${REPOPATH}" show "${NEWREF}":repos/flatcar-overlay/sys-kernel/coreos-kernel/ 2>/dev/null || git -C "${REPOPATH}" show "${NEWREF}":sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/; } | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) if [ "${FROM_KERNEL}" != "${TO_KERNEL}" ]; then KERNEL_ENTRIES=$("${SCRIPTFOLDER}"/show-fixed-kernel-cves.py -f "${FROM_KERNEL}" -t "${TO_KERNEL}") if [ "${KERNEL_ENTRIES}" != "" ]; then diff --git a/sync-with-gentoo b/sync-with-gentoo index 1a13851..8b334b2 100755 --- a/sync-with-gentoo +++ b/sync-with-gentoo @@ -1,7 +1,7 @@ #!/bin/bash # Used for syncing with gentoo. Needs to be called from the -# toplevel-directory of portage-stable. Expects the actual gentoo repo +# toplevel-directory of gentoo-subset. Expects the actual gentoo repo # to be either in ../gentoo or ../../gentoo. # # Example invocations: From 0208ebd721c0d58a85ac7e89629de5b60b6d30f7 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 19 Jul 2024 17:46:27 +0100 Subject: [PATCH 6/6] Add coreos-overlay-diff.py compatibility symlink Signed-off-by: James Le Cuirot --- coreos-overlay-diff.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 coreos-overlay-diff.py diff --git a/coreos-overlay-diff.py b/coreos-overlay-diff.py new file mode 120000 index 0000000..47c96f8 --- /dev/null +++ b/coreos-overlay-diff.py @@ -0,0 +1 @@ +flatcar-overlay-diff.py \ No newline at end of file