From 45bc34c4479b83d24e6b6ee3e68d96b7b02b30ed Mon Sep 17 00:00:00 2001 From: Thomas Gagneret Date: Thu, 18 Jan 2024 17:31:46 +0100 Subject: [PATCH] scripts: zephyr_module: Add URL, version to SPDX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve the SPDX with the current values: - URL: extracted from `git remote`. If more than one remote, URL is not set. - Version: extracted from `git rev-parse` (commit id). - PURL and CPE for Zephyr: generated from URL and version. For zephyr, the tag is extracted, if present, and replace the commit id for the version field. Since official modules does not have tags, tags are not yet extracted for modules. To track vulnerabilities from modules dependencies, a new SBOM, `modules-deps.spdx` was created. It contains the `external-references` provided by the modules. It allows to easily track vulnerabilities from these external dependencies. Signed-off-by: Thomas Gagneret (cherry picked from commit 0d05318c96ee38493e6a0411be639ebf04fe2e58) --- CMakeLists.txt | 6 +- doc/develop/modules.rst | 39 ++++ doc/develop/west/zephyr-cmds.rst | 2 + scripts/west_commands/zspdx/datatypes.py | 15 ++ scripts/west_commands/zspdx/sbom.py | 6 + scripts/west_commands/zspdx/walker.py | 142 +++++++++++--- scripts/west_commands/zspdx/writer.py | 44 ++++- scripts/zephyr_module.py | 231 ++++++++++++++++++----- 8 files changed, 406 insertions(+), 79 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f1fb89660f9c7c..28c9698eb074fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1633,9 +1633,8 @@ if(CONFIG_BUILD_OUTPUT_BIN AND CONFIG_BUILD_OUTPUT_UF2) set(BYPRODUCT_KERNEL_UF2_NAME "${PROJECT_BINARY_DIR}/${KERNEL_UF2_NAME}" CACHE FILEPATH "Kernel uf2 file" FORCE) endif() +set(KERNEL_META_PATH ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "") if(CONFIG_BUILD_OUTPUT_META) - set(KERNEL_META_PATH ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "") - list(APPEND post_build_commands COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py @@ -1649,6 +1648,9 @@ if(CONFIG_BUILD_OUTPUT_META) post_build_byproducts ${KERNEL_META_PATH} ) +else(CONFIG_BUILD_OUTPUT_META) + # Prevent spdx to use invalid data + file(REMOVE ${KERNEL_META_PATH}) endif() # Cleanup intermediate files diff --git a/doc/develop/modules.rst b/doc/develop/modules.rst index 875f6f62ff1d9f..0369a0e670c1fd 100644 --- a/doc/develop/modules.rst +++ b/doc/develop/modules.rst @@ -569,6 +569,45 @@ Build files located in a ``MODULE_EXT_ROOT`` can be described as: This allows control of the build inclusion to be described externally to the Zephyr module. +.. _modules-vulnerability-monitoring: + +Vulnerability monitoring +======================== + +The module description file :file:`zephyr/module.yml` can be used to improve vulnerability monitoring. + +If your module needs to track vulnerabilities using an external reference +(e.g your module is forked from another repository), you can use the ``security`` section. +It contains the field ``external-references`` that contains a list of references that needs to +be monitored for your module. The supported formats are: + +- CPE (Common Platform Enumeration) +- PURL (Package URL) + +.. code-block:: yaml + + security: + external-references: + - + - + - + +A real life example for `mbedTLS` module could look like this: + +.. code-block:: yaml + + security: + external-references: + - cpe:2.3:a:arm:mbed_tls:3.5.2:*:*:*:*:*:*:* + - pkg:github/Mbed-TLS/mbedtls@V3.5.2 + +.. note:: + CPE field must follow the CPE 2.3 schema provided by `NVD + `_. + PURL field must follow the PURL specification provided by `Github + `_. + + Build system integration ======================== diff --git a/doc/develop/west/zephyr-cmds.rst b/doc/develop/west/zephyr-cmds.rst index f6b2322e6a4922..1f0243c10a66d1 100644 --- a/doc/develop/west/zephyr-cmds.rst +++ b/doc/develop/west/zephyr-cmds.rst @@ -109,6 +109,8 @@ This generates the following SPDX bill-of-materials (BOM) documents in - :file:`app.spdx`: BOM for the application source files used for the build - :file:`zephyr.spdx`: BOM for the specific Zephyr source code files used for the build - :file:`build.spdx`: BOM for the built output files +- :file:`modules-deps.spdx`: BOM for modules dependencies. Check + :ref:`modules ` for more details. Each file in the bill-of-materials is scanned, so that its hashes (SHA256 and SHA1) can be recorded, along with any detected licenses if an diff --git a/scripts/west_commands/zspdx/datatypes.py b/scripts/west_commands/zspdx/datatypes.py index 1e65d96e995593..c07668cc72adb9 100644 --- a/scripts/west_commands/zspdx/datatypes.py +++ b/scripts/west_commands/zspdx/datatypes.py @@ -68,6 +68,21 @@ def __init__(self): # SPDX ID, including "SPDXRef-" self.spdxID = "" + # primary package purpose (ex. "LIBRARY", "APPLICATION", etc.) + self.primaryPurpose = "" + + # package URL + self.url = "" + + # package version + self.version = "" + + # package revision + self.revision = "" + + # package external references + self.externalReferences = [] + # the Package's declared license self.declaredLicense = "NOASSERTION" diff --git a/scripts/west_commands/zspdx/sbom.py b/scripts/west_commands/zspdx/sbom.py index fc1700817ea586..1eff541a5d59fc 100644 --- a/scripts/west_commands/zspdx/sbom.py +++ b/scripts/west_commands/zspdx/sbom.py @@ -121,4 +121,10 @@ def makeSPDX(cfg): log.err("SPDX writer failed for build document; bailing") return False + # write modules document + writeSPDX(os.path.join(cfg.spdxDir, "modules-deps.spdx"), w.docModulesExtRefs) + if not retval: + log.err("SPDX writer failed for modules-deps document; bailing") + return False + return True diff --git a/scripts/west_commands/zspdx/walker.py b/scripts/west_commands/zspdx/walker.py index 0c6469444eda07..4616c02b031c40 100644 --- a/scripts/west_commands/zspdx/walker.py +++ b/scripts/west_commands/zspdx/walker.py @@ -4,6 +4,7 @@ import os import yaml +import re from west import log from west.util import west_topdir, WestNotFound @@ -47,6 +48,7 @@ def __init__(self, cfg): self.docZephyr = None self.docApp = None self.docSDK = None + self.docModulesExtRefs = None # dict of absolute file path => the Document that owns that file self.allFileLinks = {} @@ -69,6 +71,40 @@ def __init__(self, cfg): # SDK install path from parsed CMake cache self.sdkPath = "" + def _build_purl(self, url, version=None): + if not url: + return None + + purl = None + # This is designed to match repository with the following url pattern: + # '// + COMMON_GIT_URL_REGEX=r'((git@|http(s)?:\/\/)(?P[\w\.@]+)(\/|:))(?P[\w,\-,\_]+)\/(?P[\w,\-,\_]+)(.git){0,1}((\/){0,1})$' + + match = re.fullmatch(COMMON_GIT_URL_REGEX, url) + if match: + purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}' + + if purl and (version or len(version) > 0): + purl += f'@{version}' + + return purl + + def _normalize_module_name(self, module_name): + # Replace "_" by "-" since it's not allowed in spdx ID + return module_name.replace("_", "-") + + def _add_describe_relationship(self, doc, cfgpackage): + # create DESCRIBES relationship data + rd = RelationshipData() + rd.ownerType = RelationshipDataElementType.DOCUMENT + rd.ownerDocument = doc + rd.otherType = RelationshipDataElementType.PACKAGEID + rd.otherPackageID = cfgpackage.spdxID + rd.rlnType = "DESCRIBES" + + # add it to pending relationships queue + self.pendingRelationships.append(rd) + # primary entry point def makeDocuments(self): # parse CMake cache file and get compiler path @@ -162,16 +198,7 @@ def setupAppDocument(self): pkgApp = Package(cfgPackageApp, self.docApp) self.docApp.pkgs[pkgApp.cfg.spdxID] = pkgApp - # create DESCRIBES relationship data - rd = RelationshipData() - rd.ownerType = RelationshipDataElementType.DOCUMENT - rd.ownerDocument = self.docApp - rd.otherType = RelationshipDataElementType.PACKAGEID - rd.otherPackageID = cfgPackageApp.spdxID - rd.rlnType = "DESCRIBES" - - # add it to pending relationships queue - self.pendingRelationships.append(rd) + self._add_describe_relationship(self.docApp, cfgPackageApp) def setupBuildDocument(self): # set up build document @@ -195,7 +222,7 @@ def setupBuildDocument(self): # add it to pending relationships queue self.pendingRelationships.append(rd) - def setupZephyrDocument(self, modules): + def setupZephyrDocument(self, zephyr, modules): # set up zephyr document cfgZephyr = DocumentConfig() cfgZephyr.name = "zephyr-sources" @@ -216,39 +243,67 @@ def setupZephyrDocument(self, modules): cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources" cfgPackageZephyr.relativeBaseDir = relativeBaseDir + zephyr_url = zephyr.get("remote", "") + if zephyr_url: + cfgPackageZephyr.url = zephyr_url + + if zephyr.get("revision"): + cfgPackageZephyr.revision = zephyr.get("revision") + + purl = None + zephyr_tags = zephyr.get("tags", "") + if zephyr_tags: + # Find tag vX.Y.Z + for tag in zephyr_tags: + version = re.fullmatch(r'^v(?P\d+\.\d+\.\d+)$', tag) + purl = self._build_purl(zephyr_url, tag) + + if purl: + cfgPackageZephyr.externalReferences.append(purl) + + # Extract version from tag once + if cfgPackageZephyr.version == "" and version: + cfgPackageZephyr.version = version.group('version') + + if len(cfgPackageZephyr.version) > 0: + cpe = f'cpe:2.3:o:zephyrproject:zephyr:{cfgPackageZephyr.version}:-:*:*:*:*:*:*' + cfgPackageZephyr.externalReferences.append(cpe) + pkgZephyr = Package(cfgPackageZephyr, self.docZephyr) self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr + self._add_describe_relationship(self.docZephyr, cfgPackageZephyr) + for module in modules: module_name = module.get("name", None) module_path = module.get("path", None) + module_url = module.get("remote", None) + module_revision = module.get("revision", None) if not module_name: log.err(f"cannot find module name in meta file; bailing") return False - # Replace "_" by "-" since it's not allowed in spdx ID - module_name = module_name.replace("_", "-") + module_name = self._normalize_module_name(module_name) # set up zephyr sources package cfgPackageZephyrModule = PackageConfig() - cfgPackageZephyrModule.name = module_name + cfgPackageZephyrModule.name = module_name + "-sources" cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources" cfgPackageZephyrModule.relativeBaseDir = module_path + if module_revision: + cfgPackageZephyrModule.revision = module_revision + + if module_url: + cfgPackageZephyrModule.url = module_url + pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr) self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule - # create DESCRIBES relationship data - rd = RelationshipData() - rd.ownerType = RelationshipDataElementType.DOCUMENT - rd.ownerDocument = self.docZephyr - rd.otherType = RelationshipDataElementType.PACKAGEID - rd.otherPackageID = cfgPackageZephyr.spdxID - rd.rlnType = "DESCRIBES" + self._add_describe_relationship(self.docZephyr, cfgPackageZephyrModule) - # add it to pending relationships queue - self.pendingRelationships.append(rd) + return True def setupSDKDocument(self): # set up SDK document @@ -278,6 +333,42 @@ def setupSDKDocument(self): # add it to pending relationships queue self.pendingRelationships.append(rd) + def setupModulesDocument(self, modules): + # set up zephyr document + cfgModuleExtRef = DocumentConfig() + cfgModuleExtRef.name = "modules-deps" + cfgModuleExtRef.namespace = self.cfg.namespacePrefix + "/modules-deps" + cfgModuleExtRef.docRefID = "DocumentRef-modules-deps" + self.docModulesExtRefs = Document(cfgModuleExtRef) + + for module in modules: + module_name = module.get("name", None) + module_security = module.get("security", None) + + if not module_name: + log.err(f"cannot find module name in meta file; bailing") + return False + + module_name = self._normalize_module_name(module_name) + + module_ext_ref = [] + if module_security: + module_ext_ref = module_security.get("external-references") + + # set up zephyr sources package + cfgPackageModuleExtRef = PackageConfig() + cfgPackageModuleExtRef.name = module_name + "-deps" + cfgPackageModuleExtRef.spdxID = "SPDXRef-" + module_name + "-deps" + + for ref in module_ext_ref: + cfgPackageModuleExtRef.externalReferences.append(ref) + + pkgModule = Package(cfgPackageModuleExtRef, self.docModulesExtRefs) + self.docModulesExtRefs.pkgs[pkgModule.cfg.spdxID] = pkgModule + + self._add_describe_relationship(self.docModulesExtRefs, cfgPackageModuleExtRef) + + # set up Documents before beginning def setupDocuments(self): log.dbg("setting up placeholder documents") @@ -287,7 +378,8 @@ def setupDocuments(self): try: with open(self.metaFile) as file: content = yaml.load(file.read(), yaml.SafeLoader) - self.setupZephyrDocument(content["modules"]) + if not self.setupZephyrDocument(content["zephyr"], content["modules"]): + return False except (FileNotFoundError, yaml.YAMLError): log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing") return False @@ -297,6 +389,8 @@ def setupDocuments(self): if self.cfg.includeSDK: self.setupSDKDocument() + self.setupModulesDocument(content["modules"]) + return True # walk through targets and gather information diff --git a/scripts/west_commands/zspdx/writer.py b/scripts/west_commands/zspdx/writer.py index b8afaa47953062..6883ecbd07147d 100644 --- a/scripts/west_commands/zspdx/writer.py +++ b/scripts/west_commands/zspdx/writer.py @@ -8,7 +8,16 @@ from zspdx.util import getHashes -# Output tag-value SPDX 2.2 content for the given Relationship object. +import re + +CPE23TYPE_REGEX = ( + r'^cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^' + r"`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*" + r'|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){4}$' +) +PURL_REGEX = r"^pkg:.+(\/.+)?\/.+(@.+)?(\?.+)?(#.+)?$" + +# Output tag-value SPDX 2.3 content for the given Relationship object. # Arguments: # 1) f: file handle for SPDX document # 2) rln: Relationship object being described @@ -42,7 +51,15 @@ def writeFileSPDX(f, bf): writeRelationshipSPDX(f, rln) f.write("\n") -# Output tag-value SPDX 2.2 content for the given Package object. +def generateDowloadUrl(url, revision): + # Only git is supported + # walker.py only parse revision if it's from git repositiory + if len(revision) == 0: + return url + + return f'git+{url}@{revision}' + +# Output tag-value SPDX 2.3 content for the given Package object. # Arguments: # 1) f: file handle for SPDX document # 2) pkg: Package object being described @@ -51,13 +68,34 @@ def writePackageSPDX(f, pkg): PackageName: {pkg.cfg.name} SPDXID: {pkg.cfg.spdxID} -PackageDownloadLocation: NOASSERTION PackageLicenseConcluded: {pkg.concludedLicense} """) f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense} PackageCopyrightText: {pkg.cfg.copyrightText} """) + if pkg.cfg.primaryPurpose != "": + f.write(f"PrimaryPackagePurpose: {pkg.cfg.primaryPurpose}\n") + + if len(pkg.cfg.url) > 0: + downloadUrl = generateDowloadUrl(pkg.cfg.url, pkg.cfg.revision) + f.write(f"PackageDownloadLocation: {downloadUrl}\n") + else: + f.write("PackageDownloadLocation: NOASSERTION\n") + + if len(pkg.cfg.version) > 0: + f.write(f"PackageVersion: {pkg.cfg.version}\n") + elif len(pkg.cfg.revision) > 0: + f.write(f"PackageVersion: {pkg.cfg.revision}\n") + + for ref in pkg.cfg.externalReferences: + if re.fullmatch(CPE23TYPE_REGEX, ref): + f.write(f"ExternalRef: SECURITY cpe23Type {ref}\n") + elif re.fullmatch(PURL_REGEX, ref): + f.write(f"ExternalRef: PACKAGE_MANAGER purl {ref}\n") + else: + log.wrn(f"Unknown external reference ({ref})") + # flag whether files analyzed / any files present if len(pkg.files) > 0: if len(pkg.licenseInfoFromFiles) > 0: diff --git a/scripts/zephyr_module.py b/scripts/zephyr_module.py index ab27eaa57063df..485706951e9d14 100755 --- a/scripts/zephyr_module.py +++ b/scripts/zephyr_module.py @@ -147,6 +147,15 @@ doc-url: required: false type: str + security: + required: false + type: map + mapping: + external-references: + required: false + type: seq + sequence: + - type: str ''' MODULE_YML_PATH = PurePath('zephyr/module.yml') @@ -402,24 +411,7 @@ def process_twister(module, meta): return out -def process_meta(zephyr_base, west_projs, modules, extra_modules=None, - propagate_state=False): - # Process zephyr_base, projects, and modules and create a dictionary - # with meta information for each input. - # - # The dictionary will contain meta info in the following lists: - # - zephyr: path and revision - # - modules: name, path, and revision - # - west-projects: path and revision - # - # returns the dictionary with said lists - - meta = {'zephyr': None, 'modules': None, 'workspace': None} - - workspace_dirty = False - workspace_extra = extra_modules is not None - workspace_off = False - +def _create_meta_project(project_path): def git_revision(path): rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'], stdout=subprocess.PIPE, @@ -447,77 +439,213 @@ def git_revision(path): return revision, False return None, False - zephyr_revision, zephyr_dirty = git_revision(zephyr_base) - zephyr_project = {'path': zephyr_base, - 'revision': zephyr_revision} + def git_remote(path): + popen = subprocess.Popen(['git', 'remote'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=path) + stdout, stderr = popen.communicate() + stdout = stdout.decode('utf-8') + + remotes_name = [] + if not (popen.returncode or stderr): + remotes_name = stdout.rstrip().split('\n') + + remote_url = None + + # If more than one remote, do not return any remote + if len(remotes_name) == 1: + remote = remotes_name[0] + popen = subprocess.Popen(['git', 'remote', 'get-url', remote], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=path) + stdout, stderr = popen.communicate() + stdout = stdout.decode('utf-8') + + if not (popen.returncode or stderr): + remote_url = stdout.rstrip() + + return remote_url + + def git_tags(path, revision): + if not revision or len(revision) == 0: + return None + + popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=path) + stdout, stderr = popen.communicate() + stdout = stdout.decode('utf-8') + + tags = None + if not (popen.returncode or stderr): + tags = stdout.rstrip().splitlines() + + return tags + + workspace_dirty = False + path = PurePath(project_path).as_posix() + + revision, dirty = git_revision(path) + workspace_dirty |= dirty + remote = git_remote(path) + tags = git_tags(path, revision) + + meta_project = {'path': path, + 'revision': revision} + + if remote: + meta_project['remote'] = remote + + if tags: + meta_project['tags'] = tags + + return meta_project, workspace_dirty + + +def _get_meta_project(meta_projects_list, project_path): + projects = [ prj for prj in meta_projects_list[1:] if prj["path"] == project_path ] + + return projects[0] if len(projects) == 1 else None + + +def process_meta(zephyr_base, west_projs, modules, extra_modules=None, + propagate_state=False): + # Process zephyr_base, projects, and modules and create a dictionary + # with meta information for each input. + # + # The dictionary will contain meta info in the following lists: + # - zephyr: path and revision + # - modules: name, path, and revision + # - west-projects: path and revision + # + # returns the dictionary with said lists + + meta = {'zephyr': None, 'modules': None, 'workspace': None} + + zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base) + zephyr_off = zephyr_project.get("remote") is None + + workspace_dirty = zephyr_dirty + workspace_extra = extra_modules is not None + workspace_off = zephyr_off + + if zephyr_off: + zephyr_project['revision'] += '-off' + meta['zephyr'] = zephyr_project meta['workspace'] = {} - workspace_dirty |= zephyr_dirty if west_projs is not None: from west.manifest import MANIFEST_REV_BRANCH projects = west_projs['projects'] meta_projects = [] - # Special treatment of manifest project. - manifest_proj_path = PurePath(projects[0].posixpath).as_posix() - manifest_revision, manifest_dirty = git_revision(manifest_proj_path) - workspace_dirty |= manifest_dirty - manifest_project = {'path': manifest_proj_path, - 'revision': manifest_revision} - meta_projects.append(manifest_project) - + manifest_path = projects[0].posixpath + + # Special treatment of manifest project + # Git information (remote/revision) are not provided by west for the Manifest (west.yml) + # To mitigate this, we check if we don't use the manifest from the zephyr repository or an other project. + # If it's from zephyr, reuse zephyr information + # If it's from an other project, ignore it, it will be added later + # If it's not found, we extract data manually (remote/revision) from the directory + + manifest_project = None + manifest_dirty = False + manifest_off = False + + if zephyr_base == manifest_path: + manifest_project = zephyr_project + manifest_dirty = zephyr_dirty + manifest_off = zephyr_off + elif not [ prj for prj in projects[1:] if prj.posixpath == manifest_path ]: + manifest_project, manifest_dirty = _create_meta_project( + projects[0].posixpath) + manifest_off = manifest_project.get("remote") is None + if manifest_off: + manifest_project["revision"] += "-off" + + if manifest_project: + workspace_off |= manifest_off + workspace_dirty |= manifest_dirty + meta_projects.append(manifest_project) + + # Iterates on all projects except the first one (manifest) for project in projects[1:]: - project_path = PurePath(project.posixpath).as_posix() - revision, dirty = git_revision(project_path) + meta_project, dirty = _create_meta_project(project.posixpath) workspace_dirty |= dirty - if project.sha(MANIFEST_REV_BRANCH) != revision: - revision += '-off' - workspace_off = True - meta_project = {'path': project_path, - 'revision': revision} meta_projects.append(meta_project) + off = False + if not meta_project.get("remote") or project.sha(MANIFEST_REV_BRANCH) != meta_project['revision'].removesuffix("-dirty"): + off = True + if not meta_project.get('remote') or project.url != meta_project['remote']: + # Force manifest URL and set commit as 'off' + meta_project['url'] = project.url + off = True + + if off: + meta_project['revision'] += '-off' + workspace_off |= off + + # If manifest is in project, updates related variables + if project.posixpath == manifest_path: + manifest_dirty |= dirty + manifest_off |= off + manifest_project = meta_project + meta.update({'west': {'manifest': west_projs['manifest_path'], 'projects': meta_projects}}) meta['workspace'].update({'off': workspace_off}) - meta_projects = [] + # Iterates on all modules + meta_modules = [] for module in modules: - module_path = PurePath(module.project).as_posix() - revision, dirty = git_revision(module_path) - workspace_dirty |= dirty - meta_project = {'name': module.meta['name'], - 'path': module_path, - 'revision': revision} - meta_projects.append(meta_project) - meta['modules'] = meta_projects + # Check if modules is not in projects + # It allows to have the "-off" flag since `modules` variable` does not provide URL/remote + meta_module = _get_meta_project(meta_projects, module.project) + + if not meta_module: + meta_module, dirty = _create_meta_project(module.project) + workspace_dirty |= dirty + + meta_module['name'] = module.meta.get('name') + + if module.meta.get('security'): + meta_module['security'] = module.meta.get('security') + meta_modules.append(meta_module) + + meta['modules'] = meta_modules meta['workspace'].update({'dirty': workspace_dirty, 'extra': workspace_extra}) if propagate_state: + zephyr_revision = zephyr_project['revision'] if workspace_dirty and not zephyr_dirty: zephyr_revision += '-dirty' if workspace_extra: zephyr_revision += '-extra' - if workspace_off: + if workspace_off and not zephyr_off: zephyr_revision += '-off' zephyr_project.update({'revision': zephyr_revision}) if west_projs is not None: + manifest_revision = manifest_project['revision'] if workspace_dirty and not manifest_dirty: manifest_revision += '-dirty' if workspace_extra: manifest_revision += '-extra' - if workspace_off: + if workspace_off and not manifest_off: manifest_revision += '-off' manifest_project.update({'revision': manifest_revision}) return meta -def west_projects(manifest = None): +def west_projects(manifest=None): manifest_path = None projects = [] # West is imported here, as it is optional @@ -685,7 +813,8 @@ def main(): for module in modules: kconfig += process_kconfig(module.project, module.meta) cmake += process_cmake(module.project, module.meta) - sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta) + sysbuild_kconfig += process_sysbuildkconfig( + module.project, module.meta) sysbuild_cmake += process_sysbuildcmake(module.project, module.meta) settings += process_settings(module.project, module.meta) twister += process_twister(module.project, module.meta) @@ -729,6 +858,8 @@ def main(): args.extra_modules, args.meta_state_propagate) with open(args.meta_out, 'w', encoding="utf-8") as fp: + # Ignore references and insert data instead + yaml.Dumper.ignore_aliases = lambda self, data: True fp.write(yaml.dump(meta))