Move and rename package repositories (part 2)

Catalyst 4 has totally changed the way repositories are handled. It only
works when the name of the directory containing the repository matches
the configured name of that repository. We already renamed coreos to
coreos-overlay accordingly, but we actually want entirely different
names and more convenient locations too.

The repositories are now known as gentoo-subset and flatcar-overlay, and
they live under scripts/repos. Using the same name as upstream Gentoo
would have been problematic, and just "flatcar" would have looked
awkward in documentation.

I have removed code referencing /mnt/host/source/config rather than fix
it up, as this is no location is no longer used anywhere.

Signed-off-by: James Le Cuirot <[email protected]>

Author: chewi

.github/workflows/common.sh

@@ -13,9 +13,10 @@ if [[ ! -d "${WORK_SCRIPTS_DIR:-}" ]]; then
   fail "WORK_SCRIPTS_DIR env var does not point to a directory. It should point to the scripts repo which will be updated."
 fi
 
+# Update these to use the new repo location once LTS has migrated.
 readonly SDK_OUTER_TOPDIR="${WORK_SCRIPTS_DIR}"
 readonly SDK_OUTER_OVERLAY="${SDK_OUTER_TOPDIR}/sdk_container/src/third_party/coreos-overlay"
-readonly SDK_INNER_SRCDIR="/mnt/host/source/src"
+readonly SDK_INNER_SRCDIR="/mnt/host/source/src/scripts/sdk_container/src"
 readonly SDK_INNER_OVERLAY="${SDK_INNER_SRCDIR}/third_party/coreos-overlay"
 
 readonly BUILDBOT_USERNAME="Flatcar Buildbot"

.github/workflows/portage-stable-packages-list renamed to .github/workflows/gentoo-subset-packages-list

@@ -25,8 +25,8 @@ function github_ricj_callback() {
     show_changes_env+=(
         # Override the default locations of repositories.
         "SCRIPTS_REPO=."
-        "COREOS_OVERLAY_REPO=../coreos-overlay"
-        "PORTAGE_STABLE_REPO=../portage-stable"
+        "FLATCAR_OVERLAY_REPO=../flatcar-overlay"
+        "GENTOO_SUBSET_REPO=../gentoo-subset"
     )
     show_changes_params+=(
         # We may not have a tag handy, so we tell show-changes

.github/workflows/image_changes.sh

@@ -1,4 +1,4 @@
-name: Keep portage-stable packages updated
+name: Keep gentoo-subset packages updated
 on:
   schedule:
     - cron:  '0 7 * * 1'
@@ -35,10 +35,10 @@ jobs:
           git config --global user.name "Flatcar Buildbot"
           git config --global user.email "[email protected]"
           old_head=$(git -C scripts rev-parse HEAD)
-          packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list)
+          packages_list=$(realpath scripts/.github/workflows/gentoo-subset-packages-list)
           gentoo_repo=$(realpath gentoo)
           build_scripts=$(realpath flatcar-build-scripts)
-          pushd scripts/sdk_container/src/third_party/portage-stable
+          pushd scripts/repos/gentoo-subset
           while read -r package; do
               if [[ ! -e "${package}" ]]; then
                   # If this happens, it means that the package was moved to overlay
@@ -50,7 +50,7 @@ jobs:
                   # If this happens, it means that the package was obsoleted or moved
                   # in Gentoo. The obsoletion needs to be handled in the case-by-case
                   # manner, while move should be handled by doing the same move
-                  # in portage-stable. The build should not break because of the move,
+                  # in gentoo-subset. The build should not break because of the move,
                   # because most likely it's already reflected in the profiles/updates
                   # directory.
                   echo "::warning title=${package}::Obsolete or moved package"
@@ -73,10 +73,10 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: scripts
-          branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
+          branch: buildbot/weekly-gentoo-subset-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
           delete-branch: true
           base: main
-          title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
+          title: Weekly gentoo-subset package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
           body: |
             CI: TODO
 

.github/workflows/update-portage-stable-packages-from-list.yaml renamed to .github/workflows/update-gentoo-subset-packages-from-list.yaml

@@ -15,7 +15,7 @@ jobs:
       - name: Update GLSA metadata
         id: update-glsa-metadata
         run: |
-          glsa_dir='sdk_container/src/third_party/portage-stable/metadata/glsa'
+          glsa_dir='repos/gentoo-subset/metadata/glsa'
           rm -rf "${glsa_dir}"
           mkdir --parents "${glsa_dir}"
           rsync --archive rsync://rsync.gentoo.org/gentoo-portage/metadata/glsa/* "${glsa_dir}"
@@ -30,6 +30,6 @@ jobs:
           base: main
           title: Monthly GLSA metadata ${{steps.update-glsa-metadata.outputs.TODAYDATE }}
           body: Updated GLSA metadata
-          commit-message: "portage-stable/metadata: Monthly GLSA metadata updates"
+          commit-message: "gentoo-subset/metadata: Monthly GLSA metadata updates"
           author: Flatcar Buildbot <[email protected]>
           labels: main

.github/workflows/update-metadata-glsa.yaml

@@ -35,7 +35,7 @@ Prefix uses a _staging environment_ to build binary packages, then installs thes
 The _staging environment_ contains toolchains and all build tools required to create binary packages (a full `@system`).
 The _final environment_ only contains run-time dependencies.
 
-Packages are built from ebuilds in coreos-overlay, portage-stable, and prefix-overlay.
+Packages are built from ebuilds in flatcar-overlay, gentoo-subset, and prefix-overlay.
 
 A QoL `emerge` wrapper is included to install packages to the prefix.
 

PREFIX.md

@@ -14,20 +14,18 @@ The SDK can be used to
 # Using the scripts repository
 
 The repository is meant to be the entry point for Flatcar builds and development.
-Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/):
+Ebuilds for all packages reside in one of 2 subdirectories - [flatcar-overlay](repos/flatcar-overlay) and [gentoo-subset](repos/gentoo-subset/):
 ```
 scripts
-   +--sdk_container
-          +---------src
-                     +--third_party
-                             +------coreos-overlay
-                             +------portage-stable
+   +--repos
+          +--flatcar-overlay
+          +--gentoo-subset
 ```
 
-`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
+`gentoo-subset` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
 Consider it a small sub-set of Gentoo.
 
-`coreos-overlay` contains significantly modified or even entirely self-written ebuilds.
+`flatcar-overlay` contains significantly modified or even entirely self-written ebuilds.
 
 The `scripts` repository makes ample use of tags to mark releases.
 Sometimes, local and origin tags can diverge (e.g. when re-tagging something locally to test a build).
@@ -52,7 +50,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready
 
 The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo:
 * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container.
-* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
+* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the gentoo-subset and flatcar-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
 
 ## Standalone mode
 
@@ -82,7 +80,7 @@ To start a container in privileged mode with `/dev` available use:
 
 This is the preferred mode of working with the SDK.
 Interaction with the container happens via wrapper scripts from the scripts repository.
-Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly.
+Both the host's scripts repo as well as the ebuild paths (gentoo-subset and flatcar-overlay) are made available in the container, allowing for work on these directly.
 The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency.
 
 To clone the scripts repo and pick a version:

README.md

@@ -18,13 +18,13 @@
 #     will be linked against the SEED SDK libraries, NOT against libraries built
 #     in stage 1.
 #
-# 3. stage2: Run portage-stable/scripts/bootstrap.sh
+# 3. stage2: Run gentoo-subset/scripts/bootstrap.sh
 #     This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
 #     to or otherwise influenced by whatever was in the "seed" tarball.
 #     The toolchain rebuild may contain updated package ebuilds from
-#     third_party/(portage-stable|coreos-overlay).
-#     This and all following stages use portage-stable and coreos-overlay
-#     from third_party/... (see 1.)
+#     repos/(gentoo-subset|flatcar-overlay).
+#     This and all following stages use gentoo-subset and flatcar-overlay
+#     from repos/... (see 1.)
 #
 # 4. stage3: Run emerge -e system to rebuild everything using the fresh updated
 #     toolchain from 3., using the normal USE flags provided by the profile. This
@@ -95,13 +95,13 @@ chmod 1777 "${ROOT_OVERLAY}/tmp"
 cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 
 
-# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable)
+# Stage 1 uses "known-good" ebuilds (from both flatcar-overlay and gentoo-subset)
 #  to build a minimal toolchain (USE="-*") for stage 2.
 #
-# No package updates must happen in stage 1, so we use the portage-stable and
-#  coreos-overlay paths included with the current SDK (from the SDK chroot's
-#  /var/gentoo/repos/). "Current SDK" refers to the SDK we entered with
-#  'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
+# No package updates must happen in stage 1, so we use the gentoo-subset and
+#  flatcar-overlay paths included with the current SDK (from the SDK chroot's
+#  /mnt/host/source/src/scripts/repos/). "Current SDK" refers to the SDK we
+#  entered with 'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
 #
 # Using ebuilds from the above mentioned sources will ensure that stage 1 builds
 #  a minimal stage 2 from known-good ebuild versions - the same ebuild versions

bootstrap_sdk

@@ -365,7 +365,7 @@ get_metadata() {
             local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
             if [ -n "${mirror}" ]; then
                 # Take only first mirror, those not working should be removed
-                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/scripts/repos/gentoo-subset/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
                 v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
             fi
             new_val+="${v} "
@@ -489,8 +489,8 @@ EOF
     local license_list # define before assignment because it would mask any error
     license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
     local license_dirs=(
-        "/mnt/host/source/src/third_party/coreos-overlay/licenses/"
-        "/mnt/host/source/src/third_party/portage-stable/licenses/"
+        "/mnt/host/source/src/scripts/repos/flatcar-overlay/licenses/"
+        "/mnt/host/source/src/scripts/repos/gentoo-subset/licenses/"
         "none"
     )
     for license_file in ${license_list}; do

build_library/build_image_util.sh

@@ -28,10 +28,10 @@ STAGES=
 
 DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
     "Path to directory for all catalyst images and other files."
-DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
-    "Path to the portage-stable git checkout."
-DEFINE_string coreos_overlay "${SRC_ROOT}/third_party/coreos-overlay" \
-    "Path to the coreos-overlay git checkout."
+DEFINE_string gentoo_subset "${SCRIPTS_DIR}/repos/gentoo-subset" \
+    "Path to the gentoo-subset git checkout."
+DEFINE_string flatcar_overlay "${SCRIPTS_DIR}/repos/flatcar-overlay" \
+    "Path to the flatcar-overlay git checkout."
 DEFINE_string seed_tarball "${DEFAULT_SEED}" \
     "Path to an existing stage tarball to start from."
 DEFINE_string version "${FLATCAR_VERSION}" \
@@ -62,8 +62,8 @@ storedir="$CATALYST_ROOT"
 distdir="$DISTDIR"
 envscript="$TEMPDIR/catalystrc"
 port_logdir="$CATALYST_ROOT/log"
-repo_basedir="/mnt/host/source/src/third_party"
-repo_name="portage-stable"
+repo_basedir="/mnt/host/source/src/scripts/repos"
+repo_name="gentoo-subset"
 EOF
 }
 
@@ -87,8 +87,8 @@ target: stage$1
 subarch: $ARCH
 rel_type: $TYPE
 portage_confdir: $TEMPDIR/portage
-repos: $FLAGS_coreos_overlay
-keep_repos: portage-stable coreos-overlay
+repos: $FLAGS_flatcar_overlay
+keep_repos: gentoo-subset flatcar-overlay
 profile: $FLAGS_profile
 snapshot_treeish: $FLAGS_version
 version_stamp: $FLAGS_version
@@ -231,7 +231,7 @@ write_configs() {
     info "Configuring Portage..."
     cp -r "${BUILD_LIBRARY_DIR}"/portage/ "${TEMPDIR}/"
 
-    ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \
+    ln -sfT '/mnt/host/source/src/scripts/repos/flatcar-overlay/coreos/user-patches' \
         "${TEMPDIR}"/portage/patches
 }
 
@@ -264,10 +264,10 @@ build_stage() {
 build_snapshot() {
     local repo_dir snapshot snapshots_dir snapshot_path
 
-    repo_dir=${1:-"${FLAGS_portage_stable}"}
+    repo_dir=${1:-"${FLAGS_gentoo_subset}"}
     snapshot=${2:-"${FLAGS_version}"}
     snapshots_dir="${CATALYST_ROOT}/snapshots"
-    snapshot_path="${snapshots_dir}/portage-stable-${snapshot}.sqfs"
+    snapshot_path="${snapshots_dir}/gentoo-subset-${snapshot}.sqfs"
     if [[ -f ${snapshot_path} && $FLAGS_rebuild == $FLAGS_FALSE ]]
     then
         info "Skipping snapshot, ${snapshot_path} exists"
@@ -325,5 +325,5 @@ catalyst_build() {
     fi
 
     # Cleanup snapshots, we don't use them
-    rm -rf "$CATALYST_ROOT/snapshots/${FLAGS_portage_stable##*/}-${FLAGS_version}.sqfs"*
+    rm -rf "$CATALYST_ROOT/snapshots/${FLAGS_gentoo_subset##*/}-${FLAGS_version}.sqfs"*
 }

build_library/catalyst.sh

@@ -20,11 +20,11 @@ configure_dev_portage() {
 
     # Need profiles at the bare minimum
     local repo
-    for repo in portage-stable coreos-overlay; do
+    for repo in gentoo-subset flatcar-overlay; do
         sudo mkdir -p "${root_fs_dir}/var/lib/portage/${repo}"
         sudo rsync -rtl --exclude=md5-cache \
-            "${SRC_ROOT}/third_party/${repo}/metadata" \
-            "${SRC_ROOT}/third_party/${repo}/profiles" \
+            "${SRC_ROOT}/scripts/repos/${repo}/metadata" \
+            "${SRC_ROOT}/scripts/repos/${repo}/profiles" \
             "${root_fs_dir}/var/lib/portage/${repo}"
     done
 
@@ -42,26 +42,26 @@ PORTAGE_BINHOST="$(get_binhost_url "${binhost}" "${update_group}" 'pkgs')
 $(get_binhost_url "${binhost}" "${update_group}" 'toolchain')"
 EOF
 
-    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/portage-stable.conf" <<EOF
+    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/gentoo-subset.conf" <<EOF
 [DEFAULT]
-main-repo = portage-stable
+main-repo = gentoo-subset
 
-[portage-stable]
-location = /var/lib/portage/portage-stable
+[gentoo-subset]
+location = /var/lib/portage/gentoo-subset
 EOF
 
-    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/coreos-overlay.conf" <<EOF
-[coreos-overlay]
-location = /var/lib/portage/coreos-overlay
+    sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/flatcar-overlay.conf" <<EOF
+[flatcar-overlay]
+location = /var/lib/portage/flatcar-overlay
 EOF
 
     # Now set the correct profile, we do not use the eselect tool - it
     # does not seem to be usable outside of the chroot without using
     # deprecated PORTDIR and PORTDIR_OVERLAY environment variables.
     local profile_name=$(get_board_profile "${BOARD}")
-    # Turn coreos-overlay:coreos/amd64/generic into coreos/amd64/generic/dev
+    # Turn flatcar-overlay:coreos/amd64/generic into coreos/amd64/generic/dev
     profile_name="${profile_name#*:}/dev"
-    local profile_directory="${root_fs_dir}/var/lib/portage/coreos-overlay/profiles/${profile_name}"
+    local profile_directory="${root_fs_dir}/var/lib/portage/flatcar-overlay/profiles/${profile_name}"
     if [[ ! -d "${profile_directory}" ]]; then
         die "Not a valid profile: ${profile_name}"
     fi

build_library/dev_container_util.sh

@@ -67,7 +67,7 @@ function setup_prefix_dirs() {
   sudo cp -vR "${prefix_repo}/skel/etc/portage" "${STAGINGROOT}${EPREFIX}/etc/"
   sudo cp -vR "${prefix_repo}/skel/etc/portage" "${FINALROOT}${EPREFIX}/etc/"
 
-  local profile="/mnt/host/source/src/third_party/portage-stable/profiles/default/linux"
+  local profile="/mnt/host/source/src/scripts/repos/gentoo-subset/profiles/default/linux"
   case "${PREFIX_BOARD}" in
     amd64-usr) profile="${profile}/amd64/17.1/no-multilib/prefix/kernel-3.2+";;
     arm64-usr) profile="${profile}/arm64/17.0/prefix/kernel-3.2+";;

build_library/prefix_util.sh

@@ -9,7 +9,7 @@ echo ">>> NOTICE: $0: removing 'gen-manpages', 'containerd-stress' from sysext"
 rm -f "${rootfs}/usr/bin/gen-manpages" "${rootfs}/usr/bin/containerd-stress"
 
 script_root="$(cd "$(dirname "$0")/../"; pwd)"
-files_dir="${script_root}/sdk_container/src/third_party/coreos-overlay/coreos/sysext/containerd"
+files_dir="${script_root}/repos/flatcar-overlay/coreos/sysext/containerd"
 
 echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
 cp -va "${files_dir}/"* "${rootfs}"

build_library/sysext_mangle_containerd-flatcar

@@ -8,7 +8,7 @@ echo ">>> NOTICE: $0: removing '/usr/lib/debug/', '/usr/share/docker/contrib' fr
 rm -rf "${rootfs}/usr/lib/debug/" "${rootfs}/usr/share/docker/contrib/"
 
 script_root="$(cd "$(dirname "$0")/../"; pwd)"
-files_dir="${script_root}/sdk_container/src/third_party/coreos-overlay/coreos/sysext/docker"
+files_dir="${script_root}/repos/flatcar-overlay/coreos/sysext/docker"
 
 echo ">>> NOTICE $0: installing extra files from '${files_dir}'"
 cp -va "${files_dir}/"* "${rootfs}"