diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 386253bb609e8e..3d2c6295f88f0c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update \ valgrind \ docker.io \ iputils-ping \ + icecc \ && : RUN groupadd -g $USER_GID $USERNAME \ @@ -80,3 +81,13 @@ ENV TIZEN_ROOTFS /tizen_rootfs # Fast Model GDB plugins path for debugging support ENV FAST_MODEL_PLUGINS_PATH /opt/FastModelsPortfolio_11.16/plugins/Linux64_GCC-9.3 + +# Set up ccache as a pigweed command launcher when using the scripts/build/build_examples.py +# script. Also, set up icecc as the command prefix for ccache. Such setup allows to benefit +# from compilation caching and distributed compilation at the same time. +# +# NOTE: In order to use distributed compilation with icecc, one should run +# "scripts/icecc.sh start" before starting the build. +ENV CHIP_PW_COMMAND_LAUNCHER ccache +ENV CCACHE_PREFIX icecc +ENV PATH /usr/lib/ccache:$PATH diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py index c363f6c3266f84..5f3cbbab362904 100644 --- a/scripts/build/build/__init__.py +++ b/scripts/build/build/__init__.py @@ -4,9 +4,10 @@ from enum import Enum, auto from typing import Sequence -from .targets import BUILD_TARGETS from builders.builder import BuilderOptions +from .targets import BUILD_TARGETS + class BuildSteps(Enum): GENERATED = auto() @@ -18,11 +19,12 @@ class Context: to generate make/ninja instructions and to compile. """ - def __init__(self, runner, repository_path: str, output_prefix: str): + def __init__(self, runner, repository_path: str, output_prefix: str, ninja_jobs: int): self.builders = [] self.runner = runner self.repository_path = repository_path self.output_prefix = output_prefix + self.ninja_jobs = ninja_jobs self.completed_steps = set() def SetupBuilders(self, targets: Sequence[str], options: BuilderOptions): @@ -36,7 +38,7 @@ def SetupBuilders(self, targets: Sequence[str], options: BuilderOptions): found = False for choice in BUILD_TARGETS: builder = choice.Create(target, self.runner, self.repository_path, - self.output_prefix, options) + self.output_prefix, self.ninja_jobs, options) if builder: self.builders.append(builder) found = True diff --git a/scripts/build/build/target.py b/scripts/build/build/target.py index 7f1716170178a8..78e4599fb51b75 100644 --- a/scripts/build/build/target.py +++ b/scripts/build/build/target.py @@ -389,7 +389,7 @@ def StringIntoTargetParts(self, value: str): return _StringIntoParts(value, suffix, self.fixed_targets, self.modifiers) def Create(self, name: str, runner, repository_path: str, output_prefix: str, - builder_options: BuilderOptions): + ninja_jobs: int, builder_options: BuilderOptions): parts = self.StringIntoTargetParts(name) @@ -406,6 +406,7 @@ def Create(self, name: str, runner, repository_path: str, output_prefix: str, builder.target = self builder.identifier = name builder.output_dir = os.path.join(output_prefix, name) + builder.ninja_jobs = ninja_jobs builder.chip_dir = os.path.abspath(repository_path) builder.options = builder_options diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index e1f37fca797d7f..171c788c058424 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -104,6 +104,13 @@ def ValidateTargetNames(context, parameter, values): default='./out', type=click.Path(file_okay=False, resolve_path=True), help='Prefix for the generated file output.') +@click.option( + '--ninja-jobs', + type=int, + is_flag=False, + flag_value=0, + default=None, + help='Number of ninja jobs') @click.option( '--pregen-dir', default=None, @@ -136,8 +143,8 @@ def ValidateTargetNames(context, parameter, values): 'for using ccache when building examples.')) @click.pass_context def main(context, log_level, target, enable_link_map_file, repo, - out_prefix, pregen_dir, clean, dry_run, dry_run_output, enable_flashbundle, - no_log_timestamps, pw_command_launcher): + out_prefix, ninja_jobs, pregen_dir, clean, dry_run, dry_run_output, + enable_flashbundle, no_log_timestamps, pw_command_launcher): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s %(levelname)-7s %(message)s' if no_log_timestamps: @@ -161,7 +168,7 @@ def main(context, log_level, target, enable_link_map_file, repo, logging.info('Building targets: %s', CommaSeparate(requested_targets)) context.obj = build.Context( - repository_path=repo, output_prefix=out_prefix, runner=runner) + repository_path=repo, output_prefix=out_prefix, ninja_jobs=ninja_jobs, runner=runner) context.obj.SetupBuilders(targets=requested_targets, options=BuilderOptions( enable_link_map_file=enable_link_map_file, enable_flashbundle=enable_flashbundle, diff --git a/scripts/build/builders/ameba.py b/scripts/build/builders/ameba.py index 802ce1f73cbe62..a4b3338b0cb3ad 100644 --- a/scripts/build/builders/ameba.py +++ b/scripts/build/builders/ameba.py @@ -85,8 +85,12 @@ def generate(self): title='Generating ' + self.identifier) def _build(self): - self._Execute(['ninja', '-C', self.output_dir], - title='Building ' + self.identifier) + cmd = ['ninja', '-C', self.output_dir] + + if self.ninja_jobs is not None: + cmd.append('-j' + str(self.ninja_jobs)) + + self._Execute(cmd, title='Building ' + self.identifier) def build_outputs(self): yield BuilderOutput( diff --git a/scripts/build/builders/android.py b/scripts/build/builders/android.py index 94d3a566a63877..3f640177f190ad 100644 --- a/scripts/build/builders/android.py +++ b/scripts/build/builders/android.py @@ -466,8 +466,13 @@ def _build(self): title="Building APP " + self.identifier, ) else: + cmd = ["ninja", "-C", self.output_dir] + + if self.ninja_jobs is not None: + cmd.append('-j' + str(self.ninja_jobs)) + self._Execute( - ["ninja", "-C", self.output_dir], + cmd, title="Building JNI " + self.identifier, ) diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index d2b09f393c1597..6d9385bd4fc2aa 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -95,6 +95,8 @@ def _build(self): self.PreBuildCommand() cmd = ['ninja', '-C', self.output_dir] + if self.ninja_jobs is not None: + cmd.append('-j' + str(self.ninja_jobs)) if self.build_command: cmd.append(self.build_command) diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index a07064795a1c54..8d209d6d29bf3f 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -570,7 +570,14 @@ def generate(self): def PreBuildCommand(self): if self.app == HostApp.TESTS and self.use_coverage: - self._Execute(['ninja', '-C', self.output_dir, 'default'], title="Build-only") + cmd = ['ninja', '-C', self.output_dir] + + if self.ninja_jobs is not None: + cmd.append('-j' + str(self.ninja_jobs)) + + cmd.append('default') + + self._Execute(cmd, title="Build-only") self._Execute(['lcov', '--initial', '--capture', '--directory', os.path.join(self.output_dir, 'obj'), '--exclude', os.path.join(self.chip_dir, '**/tests/*'), '--exclude', os.path.join(self.chip_dir, 'zzz_generated/*'), diff --git a/scripts/build/builders/nrf.py b/scripts/build/builders/nrf.py index 5fbc417ffe2c13..b94a695359752c 100644 --- a/scripts/build/builders/nrf.py +++ b/scripts/build/builders/nrf.py @@ -215,8 +215,12 @@ def generate(self): def _build(self): logging.info('Compiling NrfConnect at %s', self.output_dir) - self._Execute(['ninja', '-C', self.output_dir], - title='Building ' + self.identifier) + cmd = ['ninja', '-C', self.output_dir] + + if self.ninja_jobs is not None: + cmd.append('-j' + str(self.ninja_jobs)) + + self._Execute(cmd, title='Building ' + self.identifier) if self.app == NrfApp.UNIT_TESTS: # Note: running zephyr/zephyr.elf has the same result except it creates diff --git a/scripts/build/builders/telink.py b/scripts/build/builders/telink.py index 6ac4c30ca241f5..2badbce4d131eb 100644 --- a/scripts/build/builders/telink.py +++ b/scripts/build/builders/telink.py @@ -238,6 +238,9 @@ def _build(self): cmd = self.get_cmd_prefixes() + ("ninja -C %s" % self.output_dir) + if self.ninja_jobs is not None: + cmd += " -j%s" % str(self.ninja_jobs) + self._Execute(['bash', '-c', cmd], title='Building ' + self.identifier) def build_outputs(self): diff --git a/scripts/icecc.sh b/scripts/icecc.sh new file mode 100755 index 00000000000000..54783a4874b186 --- /dev/null +++ b/scripts/icecc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This scripts starts or stops the iceccd. +# It's meant to be use within devcontainer. + +if [ "$1" = "start" ]; then + IFS='.' read -ra PARTS <<<"$HOSTNAME" + if iceccd -d -m 0 -N "devcontainer-${PARTS[0]}"; then + echo "iceccd started" + else + echo "Failed to start iceccd" + fi +fi + +if [ "$1" = "stop" ]; then + if pkill icecc; then + echo "iceccd stopped" + else + echo "Failed to stop iceccd" + fi +fi