project.py: Add new 'joblib' dependency to parallelize update

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "pykwalify",
     "setuptools",
     "packaging",
+    "joblib",
 ]
 
 [project.license]

src/west/app/project.py

@@ -1,4 +1,4 @@
 # Copyright (c) 2018, 2019 Nordic Semiconductor ASA
 # Copyright 2018, 2019 Foundries.io
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -36,6 +36,12 @@
 )
 from west.manifest import is_group as is_project_group
 
+JOBLIB_PRESENT = True
+try:
+    from joblib import Parallel, delayed
+except ModuleNotFoundError:
+    JOBLIB_PRESENT = False
+
 #
 # Project-related or multi-repo commands, like "init", "update",
 # "diff", etc.
@@ -952,6 +958,12 @@
         parser.add_argument('--stats', action='store_true',
                             help='''print performance statistics for
                             update operations''')
+        if JOBLIB_PRESENT:
+            parser.add_argument('-j', '--jobs', nargs='?', const=-1,
+                                default=1, type=int, action='store',
+                                help='''Use multiple jobs to parallelize the update process.
+                                Pass -1 to use all available jobs.
+                                ''')
 
         group = parser.add_argument_group(
             title='local project clone caches',
@@ -1087,18 +1099,26 @@
             import_flags=ImportFlag.FORCE_PROJECTS)
 
         failed = []
-        for project in self.manifest.projects:
+
+        def project_update(project):
             if (isinstance(project, ManifestProject) or
                     project.name in self.updated):
-                continue
+                return
             try:
                 if not self.project_is_active(project):
                     self.dbg(f'{project.name}: skipping inactive project')
-                    continue
-                self.update(project)
+                    return
                 self.updated.add(project.name)
+                self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in self.manifest.projects:
+                project_update(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update)(project) for project in self.manifest.projects)
         self._handle_failed(self.args, failed)
 
     def update_importer(self, project, path):
@@ -1159,13 +1179,22 @@
             projects = self._projects(self.args.projects)
 
         failed = []
-        for project in projects:
+
+        def project_update_some(project):
             if isinstance(project, ManifestProject):
-                continue
+                return
             try:
                 self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in projects:
+                project_update_some(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update_some)(project) for project in projects)
+
         self._handle_failed(self.args, failed)
 
     def toplevel_projects(self):

tests/test_project.py

@@ -1,4 +1,4 @@
 # Copyright (c) 2020, Nordic Semiconductor ASA
 
 import collections
 import os
@@ -447,14 +447,14 @@
 
     assert re.search('west-commands', cmd('grep -- -- -commands'))
 
-
-def test_update_projects(west_init_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j"])
+def test_update_projects(options, west_init_tmpdir):
     # Test the 'west update' command. It calls through to the same backend
     # functions that are used for automatic updates and 'west init'
     # reinitialization.
 
     # create local repositories
-    cmd('update')
+    cmd('update ' + options)
 
     # Add commits to the local repos.
     ur = update_helper(west_init_tmpdir)
@@ -644,7 +644,8 @@
     assert modified_files.strip() == "M CODEOWNERS", \
            'local zephyr change not preserved'
 
-def test_update_some_with_imports(repos_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j -1"])
+def test_update_some_with_imports(options, repos_tmpdir):
     # 'west update project1 project2' should work fine even when
     # imports are used, as long as the relevant projects are all
     # defined in the manifest repository.
@@ -685,19 +686,19 @@
     # Updating unknown projects should fail as always.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update unknown-project', cwd=ws)
+        cmd(f'update {options} unknown-project', cwd=ws)
 
     # Updating a list of projects when some are resolved via project
     # imports must fail.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update Kconfiglib net-tools', cwd=ws)
+        cmd(f'update {options} Kconfiglib net-tools', cwd=ws)
 
     # Updates of projects defined in the manifest repository or all
     # projects must succeed, and behave the same as if no imports
     # existed.
 
-    cmd('update net-tools', cwd=ws)
+    cmd(f'update {options} net-tools', cwd=ws)
     with pytest.raises(ManifestImportFailed):
         Manifest.from_topdir(topdir=ws)
     manifest = Manifest.from_topdir(topdir=ws,
@@ -708,10 +709,10 @@
     assert net_tools_project.is_cloned()
     assert not zephyr_project.is_cloned()
 
-    cmd('update zephyr', cwd=ws)
+    cmd(f'update {options} zephyr', cwd=ws)
     assert zephyr_project.is_cloned()
 
-    cmd('update', cwd=ws)
+    cmd(f'update {options}', cwd=ws)
     manifest = Manifest.from_topdir(topdir=ws)
     assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()