Skip to content

Commit

Permalink
Get configure working again and fix the minimal build (project-chip#2…
Browse files Browse the repository at this point in the history
…8547)

* Don't bootstrap in minimal-build workflow

The configure script is a replacement for running PW bootstrap, so running
bootstrap first makes this no longer a valid test of how the configure
script sets up the build environment from scratch.

* configure: bootstrap pip from pypa.io if necessary

This is a workaround for the Debian / Ubuntu missing the ensurepip module
used by venv unless python3-venv is installed, which is "suggested" but
not required by the python3 package, and hence may or may not be installed
in build environments. (https://bugs.launchpad.net/bugs/1290847)

Also avoid := syntax which is unavailable on Python 3.7

* configure: use x64 build of zap even on arm (there is no mac arm build)

* configure: Add matter_ argument prefixes

* Revert pigweed to 0889de6~1

0889de6 introduced https://issues.pigweed.dev/issues/294886611 which breaks the
configure-based build.
  • Loading branch information
ksperling-apple authored Aug 8, 2023
1 parent 63179b0 commit 367a0c6
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 24 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/minimal-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Checkout submodules & Bootstrap
uses: ./.github/actions/checkout-submodules-and-bootstrap

- name: Checkout submodules # but don't bootstrap!
uses: ./.github/actions/checkout-submodules
with:
platform: linux

- name: Configure and build All Clusters App
run: |
CC=gcc CXX=g++ scripts/configure --project=examples/all-clusters-app/linux && ./ninja-build
38 changes: 25 additions & 13 deletions scripts/configure
Original file line number Diff line number Diff line change
Expand Up @@ -226,17 +226,32 @@ function check_build_env() {

function configure_python_env() {
progress "Setting up Python venv"
"$PYTHON" -m venv --clear "$BUILD_ENV_PATH"
# Debian and Ubuntu ship python3 with a broken venv module unless the
# python3-venv package is installed (https://bugs.launchpad.net/bugs/1290847)
local withoutpip=() pip="${BUILD_ENV_PATH}/bin/pip"
if ! "$PYTHON" -m ensurepip --version >/dev/null 2>&1; then
withoutpip=(--without-pip) pip="${pip}.pyz" # bootstrapped below
fi
"$PYTHON" -m venv --clear "${withoutpip[@]}" "$BUILD_ENV_PATH"
info "$BUILD_ENV_PATH"

# Download a standalone pip.pyz from pypa.io if necessary
if [[ -n "$withoutpip" ]]; then
progress "Bootstrapping pip via pypa.io (venv module is missing ensurepip dependency)"
call_impl download https://bootstrap.pypa.io/pip/pip.pyz "$pip"
info "ok"
fi

# Install our auto-loading venvactivate module so that running scripts via
# the venv python has the side-effect of fully activating the environment.
local sitepkgs=("${BUILD_ENV_PATH}/lib/python"*"/site-packages")
[[ -d "$sitepkgs" ]] || fail "Failed to locate venv site-packages"
cp "${CHIP_ROOT}/scripts/configure.venv/venvactivate".{pth,py} "${sitepkgs}/"

progress "Installing Python build dependencies"
"${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel
# Ensure pip and wheel are up to date first (using pip.pyz if necessary)
"${BUILD_ENV_PATH}/bin/python3" "$pip" install --require-virtualenv --quiet --upgrade pip wheel

"${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \
-r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \
-c "${CHIP_ROOT}/scripts/setup/constraints.txt"
Expand All @@ -254,19 +269,16 @@ function finalize_build_env() {
}

function download_zap() {
local version platform arch
local version
read -r version <"${CHIP_ROOT}/scripts/setup/zap.version"
case "$OSTYPE" in
linux*) platform=linux ;;
darwin*) platform=mac ;;
*) fail "Unable to determine zap platform for OSTYPE='$OSTYPE'" ;;
esac
case "$(uname -m)" in
arm64) arch=arm64 ;;
x86_64) arch=x64 ;;
*) fail "Unable to determine zap architecture for 'uname -m'='$(uname -m)'" ;;
local platform="$(uname -sm)" flavor
case "$platform" in
Linux\ x86_64) flavor=linux-x64 ;;
Linux\ arm64) flavor=linux-arm64 ;;
Darwin\ *) flavor=mac-x64 ;; # there is no mac arm build of zap (can run x64 via Rosetta)
*) fail "Unable to determine zap flavor for $platform" ;;
esac
local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${platform}-${arch}.zip"
local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${flavor}.zip"

progress "Installing zap-cli from $url"
call_impl download_and_extract_zip "$url" "${BUILD_ENV_PATH}/bin" zap-cli
Expand Down
27 changes: 19 additions & 8 deletions scripts/configure.impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
import zipfile


def download(url, dest):
urllib.request.urlretrieve(url, dest)


def download_and_extract_zip(url, dest_dir, *member_names):
file, *_ = urllib.request.urlretrieve(url)
with zipfile.ZipFile(file) as zip:
Expand All @@ -44,8 +48,10 @@ def process_project_args(gn_args_json_file, *params):

class ProjectArgProcessor:
# Prefixes to try when mapping parameters to GN arguments
BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_', 'chip_', 'chip_enable', 'chip_use_', 'chip_config_')
GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_')
BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_',
'chip_', 'chip_enable_', 'chip_use_', 'chip_config_',
'matter_', 'matter_enable_', 'matter_use_', 'matter_config_')
GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_', 'matter_', 'matter_config_')

gn_args = {} # GN arg -> type ('s'tring, 'b'ool, 'i'integer, '[' list, '{' struct)
args = collections.OrderedDict() # collected arguments
Expand Down Expand Up @@ -84,7 +90,8 @@ def add_env_arg(self, arg, envvar, default=None, list=False):
value = os.environ.get(envvar, default)
if not value:
return
if not (type := self.gn_args.get(arg, None)):
type = self.gn_args.get(arg, None)
if not type:
info("Warning: Not propagating %s, project has no build arg '%s'" % (envvar, arg))
return
self.args[arg] = json.dumps(value if type != '[' else value.split() if list else [value])
Expand All @@ -94,7 +101,8 @@ def gn_arg(self, name, prefixes=(), type=None):
arg = name.translate(str.maketrans('-', '_'))
candidates = [p + arg for p in (('',) + prefixes) if (p + arg) in self.gn_args]
preferred = [c for c in candidates if self.gn_args[c] == type] if type else []
if not (match := next(iter(preferred + candidates), None)):
match = next(iter(preferred + candidates), None)
if not match:
info("Warning: Project has no build arg '%s'" % arg)
return match

Expand All @@ -109,7 +117,8 @@ def process_triplet_parameter(self, name, value):
self.add_arg(prefix + 'os', triplet[1 if len(triplet) == 2 else 2])

def process_enable_parameter(self, name, value):
if not (arg := self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')):
arg = self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')
if not arg:
return
if self.gn_args[arg] != 'b':
fail("Project build arg '%s' is not a boolean" % arg)
Expand All @@ -118,7 +127,8 @@ def process_enable_parameter(self, name, value):
self.add_arg(arg, value is None)

def process_generic_parameter(self, name, value):
if not (arg := self.gn_arg(name, self.GENERIC_ARG_PREFIXES)):
arg = self.gn_arg(name, self.GENERIC_ARG_PREFIXES)
if not arg:
return
if self.gn_args[arg] == 'b':
fail("Project build arg '%s' is a boolean, use --enable-..." % arg)
Expand All @@ -137,9 +147,10 @@ def process_parameter(self, name, value):
def process_parameters(self, args):
"""Process GNU-style configure command line parameters"""
for arg in args:
if not (m := re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)):
match = re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)
if not match:
fail("Invalid argument: '%s'" % arg)
self.process_parameter(m.group(1), m.group(2))
self.process_parameter(match.group(1), match.group(2))


def info(message):
Expand Down
2 changes: 1 addition & 1 deletion third_party/pigweed/repo
Submodule repo updated 176 files

0 comments on commit 367a0c6

Please sign in to comment.