Get configure working again and fix the minimal build (project-chip#2…

…8547)

* Don't bootstrap in minimal-build workflow

The configure script is a replacement for running PW bootstrap, so running
bootstrap first makes this no longer a valid test of how the configure
script sets up the build environment from scratch.

* configure: bootstrap pip from pypa.io if necessary

This is a workaround for the Debian / Ubuntu missing the ensurepip module
used by venv unless python3-venv is installed, which is "suggested" but
not required by the python3 package, and hence may or may not be installed
in build environments. (https://bugs.launchpad.net/bugs/1290847)

Also avoid := syntax which is unavailable on Python 3.7

* configure: use x64 build of zap even on arm (there is no mac arm build)

* configure: Add matter_ argument prefixes

* Revert pigweed to 0889de6~1

0889de6 introduced https://issues.pigweed.dev/issues/294886611 which breaks the
configure-based build.

.github/workflows/minimal-build.yaml

@@ -36,10 +36,12 @@ jobs:
         steps:
             - name: Checkout
               uses: actions/checkout@v3
-            - name: Checkout submodules & Bootstrap
-              uses: ./.github/actions/checkout-submodules-and-bootstrap
+
+            - name: Checkout submodules # but don't bootstrap!
+              uses: ./.github/actions/checkout-submodules
               with:
                 platform: linux
+
             - name: Configure and build All Clusters App
               run: |
                   CC=gcc CXX=g++ scripts/configure --project=examples/all-clusters-app/linux && ./ninja-build

scripts/configure

@@ -226,17 +226,32 @@ function check_build_env() {
 
 function configure_python_env() {
     progress "Setting up Python venv"
-    "$PYTHON" -m venv --clear "$BUILD_ENV_PATH"
+    # Debian and Ubuntu ship python3 with a broken venv module unless the
+    # python3-venv package is installed (https://bugs.launchpad.net/bugs/1290847)
+    local withoutpip=() pip="${BUILD_ENV_PATH}/bin/pip"
+    if ! "$PYTHON" -m ensurepip --version >/dev/null 2>&1; then
+        withoutpip=(--without-pip) pip="${pip}.pyz" # bootstrapped below
+    fi
+    "$PYTHON" -m venv --clear "${withoutpip[@]}" "$BUILD_ENV_PATH"
     info "$BUILD_ENV_PATH"
 
+    # Download a standalone pip.pyz from pypa.io if necessary
+    if [[ -n "$withoutpip" ]]; then
+        progress "Bootstrapping pip via pypa.io (venv module is missing ensurepip dependency)"
+        call_impl download https://bootstrap.pypa.io/pip/pip.pyz "$pip"
+        info "ok"
+    fi
+
     # Install our auto-loading venvactivate module so that running scripts via
     # the venv python has the side-effect of fully activating the environment.
     local sitepkgs=("${BUILD_ENV_PATH}/lib/python"*"/site-packages")
     [[ -d "$sitepkgs" ]] || fail "Failed to locate venv site-packages"
     cp "${CHIP_ROOT}/scripts/configure.venv/venvactivate".{pth,py} "${sitepkgs}/"
 
     progress "Installing Python build dependencies"
-    "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel
+    # Ensure pip and wheel are up to date first (using pip.pyz if necessary)
+    "${BUILD_ENV_PATH}/bin/python3" "$pip" install --require-virtualenv --quiet --upgrade pip wheel
+
     "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \
         -r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \
         -c "${CHIP_ROOT}/scripts/setup/constraints.txt"
@@ -254,19 +269,16 @@ function finalize_build_env() {
 }
 
 function download_zap() {
-    local version platform arch
+    local version
     read -r version <"${CHIP_ROOT}/scripts/setup/zap.version"
-    case "$OSTYPE" in
-    linux*) platform=linux ;;
-    darwin*) platform=mac ;;
-    *) fail "Unable to determine zap platform for OSTYPE='$OSTYPE'" ;;
-    esac
-    case "$(uname -m)" in
-    arm64) arch=arm64 ;;
-    x86_64) arch=x64 ;;
-    *) fail "Unable to determine zap architecture for 'uname -m'='$(uname -m)'" ;;
+    local platform="$(uname -sm)" flavor
+    case "$platform" in
+    Linux\ x86_64) flavor=linux-x64 ;;
+    Linux\ arm64) flavor=linux-arm64 ;;
+    Darwin\ *) flavor=mac-x64 ;; # there is no mac arm build of zap (can run x64 via Rosetta)
+    *) fail "Unable to determine zap flavor for $platform" ;;
     esac
-    local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${platform}-${arch}.zip"
+    local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${flavor}.zip"
 
     progress "Installing zap-cli from $url"
     call_impl download_and_extract_zip "$url" "${BUILD_ENV_PATH}/bin" zap-cli

scripts/configure.impl.py

@@ -24,6 +24,10 @@
 import zipfile
 
 
+def download(url, dest):
+    urllib.request.urlretrieve(url, dest)
+
+
 def download_and_extract_zip(url, dest_dir, *member_names):
     file, *_ = urllib.request.urlretrieve(url)
     with zipfile.ZipFile(file) as zip:
@@ -44,8 +48,10 @@ def process_project_args(gn_args_json_file, *params):
 
 class ProjectArgProcessor:
     # Prefixes to try when mapping parameters to GN arguments
-    BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_', 'chip_', 'chip_enable', 'chip_use_', 'chip_config_')
-    GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_')
+    BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_',
+                         'chip_', 'chip_enable_', 'chip_use_', 'chip_config_',
+                         'matter_', 'matter_enable_', 'matter_use_', 'matter_config_')
+    GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_', 'matter_', 'matter_config_')
 
     gn_args = {}  # GN arg -> type ('s'tring, 'b'ool, 'i'integer, '[' list, '{' struct)
     args = collections.OrderedDict()  # collected arguments
@@ -84,7 +90,8 @@ def add_env_arg(self, arg, envvar, default=None, list=False):
         value = os.environ.get(envvar, default)
         if not value:
             return
-        if not (type := self.gn_args.get(arg, None)):
+        type = self.gn_args.get(arg, None)
+        if not type:
             info("Warning: Not propagating %s, project has no build arg '%s'" % (envvar, arg))
             return
         self.args[arg] = json.dumps(value if type != '[' else value.split() if list else [value])
@@ -94,7 +101,8 @@ def gn_arg(self, name, prefixes=(), type=None):
         arg = name.translate(str.maketrans('-', '_'))
         candidates = [p + arg for p in (('',) + prefixes) if (p + arg) in self.gn_args]
         preferred = [c for c in candidates if self.gn_args[c] == type] if type else []
-        if not (match := next(iter(preferred + candidates), None)):
+        match = next(iter(preferred + candidates), None)
+        if not match:
             info("Warning: Project has no build arg '%s'" % arg)
         return match
 
@@ -109,7 +117,8 @@ def process_triplet_parameter(self, name, value):
         self.add_arg(prefix + 'os', triplet[1 if len(triplet) == 2 else 2])
 
     def process_enable_parameter(self, name, value):
-        if not (arg := self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')):
+        arg = self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')
+        if not arg:
             return
         if self.gn_args[arg] != 'b':
             fail("Project build arg '%s' is not a boolean" % arg)
@@ -118,7 +127,8 @@ def process_enable_parameter(self, name, value):
         self.add_arg(arg, value is None)
 
     def process_generic_parameter(self, name, value):
-        if not (arg := self.gn_arg(name, self.GENERIC_ARG_PREFIXES)):
+        arg = self.gn_arg(name, self.GENERIC_ARG_PREFIXES)
+        if not arg:
             return
         if self.gn_args[arg] == 'b':
             fail("Project build arg '%s' is a boolean, use --enable-..." % arg)
@@ -137,9 +147,10 @@ def process_parameter(self, name, value):
     def process_parameters(self, args):
         """Process GNU-style configure command line parameters"""
         for arg in args:
-            if not (m := re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)):
+            match = re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)
+            if not match:
                 fail("Invalid argument: '%s'" % arg)
-            self.process_parameter(m.group(1), m.group(2))
+            self.process_parameter(match.group(1), match.group(2))
 
 
 def info(message):