From a1ddbbadc7f1fa8a0680a06e0d9b6215f7a52376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 18 Sep 2024 22:08:01 +0200 Subject: [PATCH 01/17] gitattributes: treat d3.js as binary Hide it from "git grep" and similar commands. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 58da6937..67e78b2d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ *.svg binary +doc/_static/d3.v4.min.js binary From 2220cf10fe30302eccbf9261cca4bd52d6df95f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 9 Oct 2024 23:35:12 +0200 Subject: [PATCH 02/17] test: add jenkins plugins smoke test Export the job configurations XML and verify that the plugin was executed successfully. --- test/black-box/jenkins-misc/config.yaml | 2 ++ test/black-box/jenkins-misc/plugins/jenkins.py | 17 +++++++++++++++++ test/black-box/jenkins-misc/run.sh | 4 ++++ 3 files changed, 23 insertions(+) create mode 100644 test/black-box/jenkins-misc/plugins/jenkins.py diff --git a/test/black-box/jenkins-misc/config.yaml b/test/black-box/jenkins-misc/config.yaml index 32d70a49..aae954ff 100644 --- a/test/black-box/jenkins-misc/config.yaml +++ b/test/black-box/jenkins-misc/config.yaml @@ -1 +1,3 @@ bobMinimumVersion: "0.20" +plugins: + - jenkins diff --git a/test/black-box/jenkins-misc/plugins/jenkins.py b/test/black-box/jenkins-misc/plugins/jenkins.py new file mode 100644 index 00000000..f3c7d12a --- /dev/null +++ b/test/black-box/jenkins-misc/plugins/jenkins.py @@ -0,0 +1,17 @@ +from xml.etree import ElementTree + +def munge(config, checkoutSteps, buildSteps, packageSteps, **kwargs): + root = ElementTree.fromstring(config) + if root.find("./properties/test.bob.canary") is None: + ElementTree.SubElement( + root.find("properties"), + "test.bob.canary") + return ElementTree.tostring(root, encoding="UTF-8") + +manifest = { + 'apiVersion' : "0.20", + 'hooks' : { + 'jenkinsJobCreate' : munge, + 'jenkinsJobPostUpdate' : munge + } +} diff --git a/test/black-box/jenkins-misc/run.sh b/test/black-box/jenkins-misc/run.sh index c0162aef..86021bba 100755 --- a/test/black-box/jenkins-misc/run.sh +++ b/test/black-box/jenkins-misc/run.sh @@ -13,6 +13,10 @@ expect_fail run_bob jenkins add local http://another.test/ run_bob jenkins graph local +# Plugins smoke test +run_bob jenkins export local "$exp" +grep -q "test.bob.canary" "$exp/root.xml" + run_bob jenkins ls run_bob jenkins ls -v run_bob jenkins ls -vv From 7a99e4f210b4d7de9f98c7ed2fcd4a971774b258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Wed, 9 Oct 2024 23:41:40 +0200 Subject: [PATCH 03/17] input: store API version with hooks Add the possibility to apply version dependent fix-ups when invoking hook functions. --- pym/bob/cmds/jenkins/jenkins.py | 4 ++-- pym/bob/input.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pym/bob/cmds/jenkins/jenkins.py b/pym/bob/cmds/jenkins/jenkins.py index c617a7ee..f3f08f60 100644 --- a/pym/bob/cmds/jenkins/jenkins.py +++ b/pym/bob/cmds/jenkins/jenkins.py @@ -1206,8 +1206,8 @@ def doJenkinsRm(recipes, argv): BobState().delJenkins(args.name) def applyHooks(hooks, job, info, reverse=False): - for h in (reversed(hooks) if reverse else hooks): - job = h(job, **info) + for hook, apiVersion in (reversed(hooks) if reverse else hooks): + job = hook(job, **info) return job def doJenkinsPush(recipes, argv): diff --git a/pym/bob/input.py b/pym/bob/input.py index 440012fe..3a46094c 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -3265,7 +3265,7 @@ def __loadPlugin(self, mangledName, fileName, name): raise ParseError("Plugin '"+fileName+"': hook name must be a string!") if not callable(fun): raise ParseError("Plugin '"+fileName+"': "+hook+": hook must be callable!") - self.__hooks.setdefault(hook, []).append(fun) + self.__hooks.setdefault(hook, []).append((fun, apiVersion)) projectGenerators = manifest.get('projectGenerators', {}) if not isinstance(projectGenerators, dict): @@ -3344,7 +3344,7 @@ def __loadPlugin(self, mangledName, fileName, name): return mod def defineHook(self, name, value): - self.__hooks[name] = [value] + self.__hooks[name] = [(value, BOB_VERSION)] def setConfigFiles(self, configFiles): self.__configFiles = configFiles @@ -3353,9 +3353,11 @@ def getCommandConfig(self): return self.__commandConfig def getHook(self, name): - return self.__hooks[name][-1] + # just return the hook function + return self.__hooks[name][-1][0] def getHookStack(self, name): + # return list of hook functions with API version return self.__hooks.get(name, []) def getProjectGenerators(self): From 8f3090b5a3b7740796bbdcae0c0bb96c8eb7abe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 00:02:05 +0200 Subject: [PATCH 04/17] input: add stablePaths path configuration knob The usage of stable paths (/bob/...) has traditionally been bound to the usage of a sandbox. This makes sense insofar as these virtual paths depend on the usage of mount namespaces. The new "stablePaths" option allows to override the usage of virtual, stable paths. By default, the existing heuristic is retained. But the usage of stable paths can be now forced or prohibited too. --- pym/bob/input.py | 120 ++++++++++++++++++++++++---------------- pym/bob/intermediate.py | 14 ++++- 2 files changed, 82 insertions(+), 52 deletions(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index 3a46094c..cc96e52b 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -451,6 +451,12 @@ def asDigestScript(self): return self.__file + " " + self.__digestSHA1 + " " + str(self.__start) + " " + str(self.__end) +class PathsConfig: + def __init__(self, pathFormatter, stablePaths): + self.pathFormatter = pathFormatter + self.stablePaths = stablePaths + + class CoreRef: """Reference from one CoreStep/CorePackage to another one. @@ -487,7 +493,7 @@ def refGetDestination(self): def refGetStack(self): return self.__stackAdd + self.__destination.refGetStack() - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): if cache is None: cache = {} if self.__diffTools: tools = inputTools.copy() @@ -499,7 +505,7 @@ def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): else: coreTool = cache.get(tool) if coreTool is None: - cache[tool] = coreTool = tool.refDeref(stack, inputTools, inputSandbox, pathFormatter, cache) + cache[tool] = coreTool = tool.refDeref(stack, inputTools, inputSandbox, pathsConfig, cache) tools[name] = coreTool else: tools = inputTools @@ -512,10 +518,10 @@ def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): sandbox = cache[self.__diffSandbox] else: sandbox = self.__diffSandbox.refDeref(stack, inputTools, inputSandbox, - pathFormatter, cache) + pathsConfig, cache) cache[self.__diffSandbox] = sandbox - return self.__destination.refDeref(stack + self.__stackAdd, tools, sandbox, pathFormatter) + return self.__destination.refDeref(stack + self.__stackAdd, tools, sandbox, pathsConfig) class CoreItem: __slots__ = [] @@ -526,7 +532,7 @@ def refGetDestination(self): def refGetStack(self): return [] - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): raise NotImplementedError @@ -600,8 +606,8 @@ def __init__(self, coreStep, path, libs, netAccess, environment, h.update(fingerprintIfStr.encode('utf8')) self.resultId = h.digest() - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): - step = self.coreStep.refDeref(stack, inputTools, inputSandbox, pathFormatter) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): + step = self.coreStep.refDeref(stack, inputTools, inputSandbox, pathsConfig) return Tool(step, self.path, self.libs, self.netAccess, self.environment, self.fingerprintScript, self.fingerprintVars) @@ -721,8 +727,8 @@ def __eq__(self, other): (self.environment == other.environment) and \ (self.user == other.user) - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): - step = self.coreStep.refDeref(stack, inputTools, inputSandbox, pathFormatter) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): + step = self.coreStep.refDeref(stack, inputTools, inputSandbox, pathsConfig) return Sandbox(step, self) class Sandbox: @@ -968,10 +974,10 @@ class Step: the step. See :meth:`bob.input.Step.getVariantId` for details. """ - def __init__(self, coreStep, package, pathFormatter): + def __init__(self, coreStep, package, pathsConfig): self._coreStep = coreStep self.__package = package - self.__pathFormatter = pathFormatter + self.__pathsConfig = pathsConfig def __repr__(self): return "Step({}, {}, {})".format(self.getLabel(), "/".join(self.getPackage().getStack()), asHexStr(self.getVariantId())) @@ -1115,10 +1121,13 @@ def getWorkspacePath(self): script but the one from getExecPath() instead. """ if self.isValid(): - return self.__pathFormatter(self, self.__package.getPluginStates()) + return self.__pathsConfig.pathFormatter(self, self.__package.getPluginStates()) else: return "/invalid/workspace/path/of/{}".format(self.__package.getName()) + def stablePaths(self): + return self.__pathsConfig.stablePaths + def getTools(self): """Get dictionary of tools. @@ -1139,7 +1148,7 @@ def getArguments(self): p = self.__package refCache = {} return [ a.refDeref(p.getStack(), p._getInputTools(), p._getInputSandboxRaw(), - self.__pathFormatter, refCache) + self.__pathsConfig, refCache) for a in self._coreStep.args ] def getAllDepSteps(self): @@ -1183,7 +1192,7 @@ def _getProvidedDeps(self): p = self.__package refCache = {} return [ a.refDeref(p.getStack(), p._getInputTools(), p._getInputSandboxRaw(), - self.__pathFormatter, refCache) + self.__pathsConfig, refCache) for a in self._coreStep.providedDeps ] def _isFingerprinted(self): @@ -1280,9 +1289,9 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[], deterministic = corePackage.recipe.checkoutDeterministic super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, toolDepWeak) - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): - package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathFormatter) - ret = CheckoutStep(self, package, pathFormatter) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): + package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) + ret = CheckoutStep(self, package, pathsConfig) package._setCheckoutStep(ret) return ret @@ -1368,9 +1377,9 @@ def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), self.fingerprintMask = fingerprintMask super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): - package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathFormatter) - ret = BuildStep(self, package, pathFormatter) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): + package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) + ret = BuildStep(self, package, pathsConfig) package._setBuildStep(ret) return ret @@ -1405,9 +1414,9 @@ def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env= self.fingerprintMask = fingerprintMask super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): - package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathFormatter) - ret = PackageStep(self, package, pathFormatter) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): + package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) + ret = PackageStep(self, package, pathsConfig) package._setPackageStep(ret) return ret @@ -1447,7 +1456,7 @@ def hasNetAccess(self): class CorePackageInternal(CoreItem): __slots__ = [] - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter, cache=None): + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): return (inputTools, inputSandbox) corePackageInternal = CorePackageInternal() @@ -1469,9 +1478,9 @@ def __init__(self, recipe, tools, diffTools, sandbox, diffSandbox, self.pkgId = pkgId self.metaEnv = metaEnv - def refDeref(self, stack, inputTools, inputSandbox, pathFormatter): - tools, sandbox = self.internalRef.refDeref(stack, inputTools, inputSandbox, pathFormatter) - return Package(self, stack, pathFormatter, inputTools, tools, inputSandbox, sandbox) + def refDeref(self, stack, inputTools, inputSandbox, pathsConfig): + tools, sandbox = self.internalRef.refDeref(stack, inputTools, inputSandbox, pathsConfig) + return Package(self, stack, pathsConfig, inputTools, tools, inputSandbox, sandbox) def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic, @@ -1523,10 +1532,10 @@ class Package(object): package. """ - def __init__(self, corePackage, stack, pathFormatter, inputTools, tools, inputSandbox, sandbox): + def __init__(self, corePackage, stack, pathsConfig, inputTools, tools, inputSandbox, sandbox): self.__corePackage = corePackage self.__stack = stack - self.__pathFormatter = pathFormatter + self.__pathsConfig = pathsConfig self.__inputTools = inputTools self.__tools = tools self.__inputSandbox = inputSandbox @@ -1586,7 +1595,7 @@ def getDirectDepSteps(self): """ refCache = {} return [ d.refDeref(self.__stack, self.__inputTools, self.__inputSandbox, - self.__pathFormatter, refCache) + self.__pathsConfig, refCache) for d in self.__corePackage.directDepSteps ] def getIndirectDepSteps(self): @@ -1597,7 +1606,7 @@ def getIndirectDepSteps(self): """ refCache = {} return [ d.refDeref(self.__stack, self.__inputTools, self.__inputSandbox, - self.__pathFormatter, refCache) + self.__pathsConfig, refCache) for d in self.__corePackage.indirectDepSteps ] def getAllDepSteps(self): @@ -1622,7 +1631,7 @@ def getCheckoutStep(self): ret = self.__checkoutStep except AttributeError: ret = self.__checkoutStep = CheckoutStep(self.__corePackage.checkoutStep, - self, self.__pathFormatter) + self, self.__pathsConfig) return ret def _setBuildStep(self, buildStep): @@ -1634,7 +1643,7 @@ def getBuildStep(self): ret = self.__buildStep except AttributeError: ret = self.__buildStep = BuildStep(self.__corePackage.buildStep, - self, self.__pathFormatter) + self, self.__pathsConfig) return ret def _setPackageStep(self, packageStep): @@ -1646,7 +1655,7 @@ def getPackageStep(self): ret = self.__packageStep except AttributeError: ret = self.__packageStep = PackageStep(self.__corePackage.packageStep, - self, self.__pathFormatter) + self, self.__pathsConfig) return ret def getPluginStates(self): @@ -3787,7 +3796,7 @@ def getClass(self, className): raise ParseError("Class {} requested but not found.".format(className)) return self.__classes[className] - def __generatePackages(self, nameFormatter, cacheKey, sandboxEnabled): + def __generatePackages(self, pathsConfig, cacheKey, sandboxEnabled): # use separate caches with and without sandbox if sandboxEnabled: cacheName = ".bob-packages-sb.pickle" @@ -3800,8 +3809,8 @@ def __generatePackages(self, nameFormatter, cacheKey, sandboxEnabled): persistedCacheKey = f.read(len(cacheKey)) if cacheKey == persistedCacheKey: tmp = PackageUnpickler(f, self.getRecipe, self.__plugins, - nameFormatter).load() - return tmp.refDeref([], {}, None, nameFormatter) + pathsConfig).load() + return tmp.refDeref([], {}, None, pathsConfig) except FileNotFoundError: pass except Exception as e: @@ -3816,14 +3825,27 @@ def __generatePackages(self, nameFormatter, cacheKey, sandboxEnabled): newCacheName = cacheName + ".new" with open(newCacheName, "wb") as f: f.write(cacheKey) - PackagePickler(f, nameFormatter).dump(result) + PackagePickler(f, pathsConfig).dump(result) replacePath(newCacheName, cacheName) except OSError as e: Warn("Could not save package cache: " + str(e)).show(cacheName) - return result.refDeref([], {}, None, nameFormatter) + return result.refDeref([], {}, None, pathsConfig) - def generatePackages(self, nameFormatter, sandboxEnabled=False): + def generatePackages(self, nameFormatter, sandboxEnabled=False, stablePaths=None): + """Generate package set. + + :param nameFormatter: Function returning path for a given step. + :type nameFormatter: Callable[[Step, Mapping[str, PluginState]], str] + :param sandboxEnabled: Enable sandbox image dependencies. + :type sandboxEnabled: bool + :param stablePaths: Configure usage of stable execution paths (/bob/...). + * ``None``: Use stable path in sandbox image, otherwise workspace path. + * ``False``: Always use workspace path. + * ``True``: Always use stable path (/bob/...). + :type stablePaths: None | bool + """ + pathsConfig = PathsConfig(nameFormatter, stablePaths) # calculate cache key for persisted packages h = hashlib.sha1() h.update(BOB_INPUT_HASH) @@ -3836,7 +3858,7 @@ def generatePackages(self, nameFormatter, sandboxEnabled=False): cacheKey = h.digest() return PackageSet(cacheKey, self.__aliases, self.__stringFunctions, - lambda: self.__generatePackages(nameFormatter, cacheKey, sandboxEnabled), + lambda: self.__generatePackages(pathsConfig, cacheKey, sandboxEnabled), self._queryMode or self.__uiConfig.get('queryMode', 'nullglob')) def getPolicy(self, name, location=None): @@ -3955,29 +3977,29 @@ def loadBinary(self, name): class PackagePickler(pickle.Pickler): - def __init__(self, file, pathFormatter): + def __init__(self, file, pathsConfig): super().__init__(file, -1, fix_imports=False) - self.__pathFormatter = pathFormatter + self.__pathsConfig = pathsConfig def persistent_id(self, obj): - if obj is self.__pathFormatter: - return ("pathfmt", None) + if obj is self.__pathsConfig: + return ("pathscfg", None) elif isinstance(obj, Recipe): return ("recipe", obj.getPackageName()) else: return None class PackageUnpickler(pickle.Unpickler): - def __init__(self, file, recipeGetter, plugins, pathFormatter): + def __init__(self, file, recipeGetter, plugins, pathsConfig): super().__init__(file) self.__recipeGetter = recipeGetter self.__plugins = plugins - self.__pathFormatter = pathFormatter + self.__pathsConfig = pathsConfig def persistent_load(self, pid): (tag, key) = pid - if tag == "pathfmt": - return self.__pathFormatter + if tag == "pathscfg": + return self.__pathsConfig elif tag == "recipe": return self.__recipeGetter(key) else: diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 0b0d3236..44efb259 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -64,6 +64,7 @@ def fromStep(cls, step, graph, partial=False): self.__data['isRelocatable'] = step.isRelocatable() self.__data['isShared'] = step.isShared() self.__data['sandbox'] = graph.addSandbox(step.getSandbox()) + self.__data['stablePaths'] = step.stablePaths() if not partial: self.__data['isFingerprinted'] = step._isFingerprinted() @@ -144,6 +145,9 @@ def isShared(self): def getWorkspacePath(self): return self.__data['workspacePath'] + def stablePaths(self): + return self.__data['stablePaths'] + def getExecPath(self, referrer=None): """Return the execution path of the step. @@ -153,10 +157,14 @@ def getExecPath(self, referrer=None): to this step while building. """ if self.isValid(): - if (referrer or self).getSandbox() is None: - return self.getStoragePath() - else: + stablePaths = self.stablePaths() + if stablePaths is None: + stablePaths = (referrer or self).getSandbox() is not None + + if stablePaths: return os.path.join("/bob", asHexStr(self.getVariantId()), "workspace") + else: + return self.getStoragePath() else: return "/invalid/exec/path/of/{}".format(self.getPackage().getName()) From 3e942c8e134a6fa5d0e0ddd8ef7b16f85ce68254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 00:21:12 +0200 Subject: [PATCH 05/17] builder: add "slim sandbox" build mode The slim sandbox uses the bob-namespace-sandbox to restrict access to workspace paths. But in contrast to the existing "fat" sandbox, a slim sandbox does not use a sandbox image. To gain at least a bit of information hiding, the current working directory is hidden behind a whiteout. This should usually hide all other workspaces. --- pym/bob/builder.py | 13 +++++++--- pym/bob/invoker.py | 55 +++++++++++++++++++++++++++++++++--------- pym/bob/languages.py | 57 ++++++++++++++++++++++++++------------------ 3 files changed, 88 insertions(+), 37 deletions(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index e7bd85b6..d5b7a55c 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -407,6 +407,7 @@ def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv, self.__installSharedPackages = False self.__executor = None self.__attic = True + self.__slimSandbox = False def setExecutor(self, executor): self.__executor = executor @@ -505,6 +506,9 @@ def setAuditMeta(self, keys): def setAtticEnable(self, enable): self.__attic = enable + def setSlimSandbox(self, enable): + self.__slimSandbox = enable + def setShareHandler(self, handler): self.__share = handler @@ -737,7 +741,8 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None, logFile = os.path.join(workspacePath, "..", "log.txt") scriptHint = os.path.join(workspacePath, "..", "script") spec = StepSpec.fromStep(step, envFile, self.__envWhiteList, logFile, - scriptHint=scriptHint, isJenkins=step.JENKINS) + scriptHint=scriptHint, isJenkins=step.JENKINS, + slimSandbox=self.__slimSandbox) # Write invocation wrapper except on Jenkins. There will be nobody # around to actually execute it... @@ -1934,7 +1939,8 @@ async def __calcFingerprintTask(self, step, sandbox, key, depth): return fingerprint async def __runFingerprintScript(self, step, logger): - spec = StepSpec.fromStep(step, None, self.__envWhiteList, isJenkins=step.JENKINS) + spec = StepSpec.fromStep(step, None, self.__envWhiteList, isJenkins=step.JENKINS, + slimSandbox=self.__slimSandbox) invoker = Invoker(spec, self.__preserveEnv, True, self.__verbose >= INFO, self.__verbose >= NORMAL, self.__verbose >= DEBUG, self.__bufferedStdIO, @@ -1952,7 +1958,8 @@ async def __runFingerprintScript(self, step, logger): async def __runScmSwitch(self, step, scmPath, scm, oldSpec): logFile = os.path.join(step.getWorkspacePath(), "..", "log.txt") - spec = StepSpec.fromStep(step, None, self.__envWhiteList, logFile, isJenkins=step.JENKINS) + spec = StepSpec.fromStep(step, None, self.__envWhiteList, logFile, isJenkins=step.JENKINS, + slimSandbox=self.__slimSandbox) invoker = Invoker(spec, self.__preserveEnv, self.__noLogFile, self.__verbose >= INFO, self.__verbose >= NORMAL, self.__verbose >= DEBUG, self.__bufferedStdIO, diff --git a/pym/bob/invoker.py b/pym/bob/invoker.py index c3204c3d..ee87cb45 100644 --- a/pym/bob/invoker.py +++ b/pym/bob/invoker.py @@ -267,7 +267,7 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None, return FinishedProcess(ret, stdoutBuf, stderrBuf) - def __getSandboxCmds(self, tmpDir): + def __getFatSandboxCmds(self, tmpDir): if sys.platform != "linux": self.fail("Sandbox builds are only supported on Linux!") @@ -303,6 +303,31 @@ def __getSandboxCmds(self, tmpDir): return cmdArgs + def __getSlimSandboxCmds(self, tmpDir): + if sys.platform != "linux": + self.fail("Sandbox builds are only supported on Linux!") + + sandboxTmpDir = os.path.join(tmpDir, "sandbox") + os.mkdir(sandboxTmpDir) + whiteoutTmpDir = os.path.join(tmpDir, "whiteout") + os.mkdir(whiteoutTmpDir) + + cmdArgs = [ self.__getSandboxHelperPath() ] + cmdArgs.extend(["-S", sandboxTmpDir]) + cmdArgs.append("-i") + cmdArgs.extend(["-d", "/tmp"]) + + # Mount everything read-only except tmp + for f in os.listdir("/"): + if f == "tmp": continue + cmdArgs.extend(["-M", "/"+f, "-m", "/"+f]) + + # Whiteout current directory. Other workspaces should only be visible + # if they are a depdendency. + cmdArgs.extend(["-M", whiteoutTmpDir, "-w", os.getcwd()]) + + return cmdArgs + async def executeStep(self, mode, clean=False, keepSandbox=False): # make permissions predictable os.umask(0o022) @@ -349,12 +374,17 @@ async def executeStep(self, mode, clean=False, keepSandbox=False): # Wrap call into sandbox if requested if self.__spec.hasSandbox: - cmdArgs = self.__getSandboxCmds(tmpDir) + if self.__spec.fatSandbox: + cmdArgs = self.__getFatSandboxCmds(tmpDir) + else: + assert self.__spec.slimSandbox + cmdArgs = self.__getSlimSandboxCmds(tmpDir) + cmdArgs.extend(["-M", os.path.abspath(realScriptFile), "-m" , execScriptFile]) # Prevent network access - if not self.__spec.sandboxNetAccess: cmdArgs.append('-n') + if not self.__spec.netAccess: cmdArgs.append('-n') # Create empty env file. Otheriwse bind mount fails. if self.__spec.envFile: @@ -363,17 +393,20 @@ async def executeStep(self, mode, clean=False, keepSandbox=False): # Mount workspace writable and all dependencies read-only cmdArgs.extend(["-M", os.path.abspath(self.__spec.workspaceWorkspacePath), - "-w", self.__spec.workspaceExecPath]) + "-w", os.path.abspath(self.__spec.workspaceExecPath)]) cmdArgs.extend(["-W", os.path.abspath(self.__spec.workspaceExecPath) ]) - for argWorkspacePath, argExecPath in self.__spec.sandboxDepMounts: + for argWorkspacePath, argExecPath in self.__spec.depMounts: cmdArgs.extend(["-M", os.path.abspath(argWorkspacePath), - "-m", argExecPath]) + "-m", os.path.abspath(argExecPath)]) # Command follows. Stop parsing options. cmdArgs.append("--") # Hard override of PATH. The sandbox helper is found by an absolute path! - env = { "PATH" : ":".join(self.__spec.sandboxPaths) } + if self.__spec.fatSandbox: + env = { "PATH" : ":".join(self.__spec.sandboxPaths) } + else: + env = None else: cmdArgs = [] env = None @@ -443,12 +476,12 @@ async def executeFingerprint(self, keepSandbox=False): # Wrap call into sandbox if requested env = {} - if self.__spec.hasSandbox: + if self.__spec.fatSandbox: env["BOB_CWD"] = "/bob/fingerprint" env["PATH"] = ":".join(self.__spec.sandboxPaths) # Setup workspace - cmdArgs = self.__getSandboxCmds(tmpDir) + cmdArgs = self.__getFatSandboxCmds(tmpDir) cmdArgs.extend(["-d", "/bob/fingerprint"]) cmdArgs.extend(["-W", "/bob/fingerprint"]) @@ -474,12 +507,12 @@ async def executeFingerprint(self, keepSandbox=False): raise BuildError(e.what) finally: if tmpDir is not None: - if not self.__spec.hasSandbox or not keepSandbox: + if not self.__spec.fatSandbox or not keepSandbox: try: removePath(tmpDir) except OSError as e: self.error("Error removing sandbox:", str(e)) - elif self.__spec.hasSandbox: + elif self.__spec.fatSandbox: self.info("Keeping sandbox image at", tmpDir) self.__closeLog(ret) diff --git a/pym/bob/languages.py b/pym/bob/languages.py index 7e509bd2..9be69b71 100644 --- a/pym/bob/languages.py +++ b/pym/bob/languages.py @@ -362,7 +362,7 @@ def __formatScript(spec, script): @staticmethod def __scriptFilePaths(spec, tmpDir): - if spec.hasSandbox: + if spec.fatSandbox: execScriptFile = "/.script" realScriptFile = spec.scriptHint or os.path.join(tmpDir, ".script") else: @@ -590,7 +590,7 @@ def __formatScript(spec, script, trace): @staticmethod def __scriptFilePaths(spec, tmpDir): - if spec.hasSandbox: + if spec.fatSandbox: execScriptFile = "/.script.ps1" realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".ps1" else: @@ -670,7 +670,7 @@ class StepSpec: @classmethod def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=False, - scriptHint=None): + scriptHint=None, slimSandbox=False): self = cls() scriptLanguage = step.getPackage().getRecipe().scriptLanguage self.__data = d = { @@ -679,6 +679,7 @@ def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=F 'logFile' : logFile, 'isJenkins' : isJenkins, 'scriptHint' : scriptHint, + 'slimSandbox' : slimSandbox, 'vsn' : asHexStr(BOB_INPUT_HASH), 'language' : scriptLanguage.index.value, 'env' : dict(step.getEnv()), @@ -698,6 +699,7 @@ def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=F (n, os.path.join(t.getStep().getExecPath(step), t.getPath())) for (n,t) in step.getTools().items() ]), + 'netAccess' : step.hasNetAccess(), } if step.isCheckoutStep(): @@ -714,23 +716,24 @@ def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=F 'paths' : step.getSandbox().getPaths(), 'hostMounts' : step.getSandbox().getMounts(), 'user' : step.getSandbox().getUser(), - 'netAccess' : step.hasNetAccess(), - 'depMounts' : [ - (dep.getStoragePath(), dep.getExecPath(step)) - for dep in step.getAllDepSteps() if dep.isValid() - ], } - # Special handling to mount all previous steps of current package. - # It is defined that the checkout and build step are visible in the - # sandbox for a given package step. We must stop at checkout steps - # because they might have dependencies due to the 'checkoutDep' - # flag. - extra = step - while extra.isValid() and not extra.isCheckoutStep() and len(extra.getArguments()) > 0: - extra = extra.getArguments()[0] - if extra.isValid(): - s['depMounts'].append((extra.getStoragePath(), extra.getExecPath(step))) + # What needs to be mounted in a user namespace slim/fat sandbox + d['depMounts'] = depMounts = [ + (dep.getStoragePath(), dep.getExecPath(step)) + for dep in step.getAllDepSteps() if dep.isValid() + ] + + # Special handling to mount all previous steps of current package. + # It is defined that the checkout and build step are visible in the + # sandbox for a given package step. We must stop at checkout steps + # because they might have dependencies due to the 'checkoutDep' + # flag. + extra = step + while extra.isValid() and not extra.isCheckoutStep() and len(extra.getArguments()) > 0: + extra = extra.getArguments()[0] + if extra.isValid(): + depMounts.append((extra.getStoragePath(), extra.getExecPath(step))) d['preRunCmds'] = step.getJenkinsPreRunCmds() if isJenkins else step.getPreRunCmds() d['setupScript'] = step.getSetupScript() @@ -758,9 +761,17 @@ def toString(self): return json.dumps(self.__data, indent="\t", sort_keys=True) @property - def hasSandbox(self): + def fatSandbox(self): return 'sandbox' in self.__data + @property + def slimSandbox(self): + return self.__data['slimSandbox'] + + @property + def hasSandbox(self): + return self.fatSandbox or self.slimSandbox + @property def language(self): return getLanguage(ScriptLanguage(self.__data['language'])) @@ -786,12 +797,12 @@ def paths(self): return self.__data['paths'] @property - def sandboxDepMounts(self): - return self.__data['sandbox']['depMounts'] + def depMounts(self): + return self.__data['depMounts'] @property - def sandboxNetAccess(self): - return self.__data['sandbox']['netAccess'] + def netAccess(self): + return self.__data['netAccess'] @property def sandboxRootWorkspace(self): From 9dd6ac788162c19766028edbc247607a3102ffcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 22:20:18 +0200 Subject: [PATCH 06/17] utils: add SandboxMode helper class To introduce different sandbox modes, a SandboxMode helper is added. This helper controls three different aspects of the sandbox operation: * slimSandbox: use mount namespaces without a sandbox image. * sandboxEnabled: usage of the sanbox image * stablePaths: force or prohibit stable paths Using these properties, the following sandbox modes are defined: * no-sandbox * sandbox: traditional sandbox mode * slim-sandbox: always isolate, always use workspace path, not use sandbox images. * dev-sandbox: always isolate, always use workspace path, use sandbox images if available. * strict-sandbox: always isolate, always use stable paths, use sandbox images if available. --- pym/bob/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pym/bob/utils.py b/pym/bob/utils.py index d6677344..9f04d44a 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -97,6 +97,34 @@ def removeUserFromUrl(url): # Nothing matched return url +class SandboxMode: + def __init__(self, mode): + # normalize legacy boolean argument + if isinstance(mode, bool): + mode = "yes" if mode else "no" + assert mode in ("no", "yes", "slim", "dev", "strict") + + self.mode = mode + self.sandboxEnabled = mode in ("dev", "yes", "strict") + self.slimSandbox = mode in ("slim", "dev", "strict") + + if mode == "dev": + self.stablePaths = False + elif mode == "strict": + self.stablePaths = True + else: + self.stablePaths = None + + @property + def compatMode(self): + # pre 0.25 compatibility + if self.mode == "no": + return False + elif self.mode == "yes": + return True + else: + return self.mode + def removePath(path): if sys.platform == "win32": def onerror(func, path, exc): From 145596bf20daa94a3b315448f325211608c933c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 22:41:48 +0200 Subject: [PATCH 07/17] build: add support for new sandbox modes Adds the new --slim-sandbox, --dev-sandbox and --strict-sandbox options to the dev/build commands. --- pym/bob/cmds/build/build.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pym/bob/cmds/build/build.py b/pym/bob/cmds/build/build.py index d590f723..61397ebf 100644 --- a/pym/bob/cmds/build/build.py +++ b/pym/bob/cmds/build/build.py @@ -12,7 +12,7 @@ from ...layers import updateLayers from ...share import getShare from ...tty import setVerbosity, setTui, Warn -from ...utils import copyTree, processDefines, EventLoopWrapper +from ...utils import copyTree, processDefines, EventLoopWrapper, SandboxMode import argparse import datetime import re @@ -197,16 +197,25 @@ def _downloadLayerArgument(arg): help="Use shared packages") group.add_argument('--no-shared', action='store_false', dest='shared', help="Do not use shared packages") + group = parser.add_mutually_exclusive_group() group.add_argument('--install', action='store_true', default=None, help="Install shared packages") group.add_argument('--no-install', action='store_false', dest='install', help="Do not install shared packages") + group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=None, - help="Enable sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', + group.add_argument('--sandbox', action='store_const', const="yes", default=None, + help="Enable partial sandboxing") + group.add_argument('--slim-sandbox', action='store_const', const="slim", dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_const', const="dev", dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_const', const="strict", dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_const', const="no", dest='sandbox', help="Disable sandboxing") + parser.add_argument('--clean-checkout', action='store_true', default=None, dest='clean_checkout', help="Do a clean checkout if SCM state is dirty.") group = parser.add_mutually_exclusive_group() @@ -247,7 +256,7 @@ def _downloadLayerArgument(arg): 'clean' : not develop, 'upload' : False, 'download' : "deps" if develop else "yes", - 'sandbox' : not develop, + 'sandbox' : "no" if develop else "yes", 'clean_checkout' : False, 'no_logfiles' : False, 'link_deps' : True, @@ -299,7 +308,10 @@ def _downloadLayerArgument(arg): nameFormatter = recipes.getHook('releaseNameFormatter') nameFormatter = LocalBuilder.releaseNamePersister(nameFormatter) nameFormatter = LocalBuilder.makeRunnable(nameFormatter) - packages = recipes.generatePackages(nameFormatter, args.sandbox) + + sandboxMode = SandboxMode(args.sandbox) + packages = recipes.generatePackages(nameFormatter, sandboxMode.sandboxEnabled, + sandboxMode.stablePaths) if develop: developPersister.prime(packages) verbosity = cfg.get('verbosity', 0) + args.verbose - args.quiet @@ -325,6 +337,7 @@ def _downloadLayerArgument(arg): builder.setShareHandler(getShare(recipes.getShareConfig())) builder.setShareMode(args.shared, args.install) builder.setAtticEnable(args.attic) + builder.setSlimSandbox(sandboxMode.slimSandbox) if args.resume: builder.loadBuildState() backlog = [] From 6c2a4c0c72bd676c4f21605a71b7217235162918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 22:49:35 +0200 Subject: [PATCH 08/17] input: support new sandbox modes in command defaults Let the "sandbox" command default additionally accept the 'no', 'yes', 'slim', 'dev' and 'strict' strings that correspond to the respective command line options. --- pym/bob/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index cc96e52b..60e355fe 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -2940,7 +2940,7 @@ class RecipeSet: schema.Optional('link_deps') : bool, schema.Optional('no_deps') : bool, schema.Optional('no_logfiles') : bool, - schema.Optional('sandbox') : bool, + schema.Optional('sandbox') : schema.Or(bool, "no", "slim", "dev", "yes", "strict"), schema.Optional('shared') : bool, schema.Optional('upload') : bool, schema.Optional('verbosity') : int, From 0f1b118b6ebda59fbb452a26cab2fc19e76f538b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 23:45:28 +0200 Subject: [PATCH 09/17] doc: document new dev/build sandbox modes --- doc/manpages/bob-build-dev.rst | 90 +++++++++++++++++++++++++++++++++- doc/manpages/bob-build.rst | 3 +- doc/manpages/bob-dev.rst | 4 +- doc/manual/configuration.rst | 5 +- 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/doc/manpages/bob-build-dev.rst b/doc/manpages/bob-build-dev.rst index eebae90e..a3a0de94 100644 --- a/doc/manpages/bob-build-dev.rst +++ b/doc/manpages/bob-build-dev.rst @@ -7,6 +7,67 @@ be initialized with :ref:`bob init `. Any number of build trees may refer to the same project. Inside the external build-tree there may be a dedicated ``default.yaml``, overriding settings from the project. +Sandboxing +---------- + +Sandboxing allows to execute the build steps in ephemeral containers. The +feature is currently available on Linux only. There are different aspects to +sandboxing: + +1. Isolating from the host environment. By using a project defined sandbox + image, the build environment is made independent of the host Linux + distribution. +2. Controlling the accessible project paths. Only declared dependencies are + accessible read-only. The build workspace is the only writable path (despite + ``/tmp``). All other project paths are not not accessible at all. +3. Providing stable execution paths. Sometimes the build path is leaking into + the created binaries. Inside the sandbox environment, the paths can be made + reproducible. + +.. only:: not man + + To accommodate for different use cases, five different sandbox modes are + supported by Bob. They differ in their degree of isolation and execution path + stability: + + +----------------------+--------------------------------+------------------------------------------+ + | Mode | Packages without sandbox image | Packages with sandbox image | + | +-------------+------------------+-----------+-------------+----------------+ + | | Isolation | Execution path | Isolation | Image used? | Execution path | + +======================+=============+==================+===========+=============+================+ + | ``--no-sandbox`` | \- | Workspace | \- | n/a | Workspace | + +----------------------+-------------+------------------+-----------+-------------+----------------+ + | ``--sandbox`` | \- | Workspace | Yes | Yes | Stable | + +----------------------+-------------+------------------+-----------+-------------+----------------+ + | ``--slim-sandbox`` | Yes | Workspace | Yes | \- | Workspace | + +----------------------+-------------+------------------+-----------+-------------+----------------+ + | ``--dev-sandbox`` | Yes | Workspace | Yes | Yes | Workspace | + +----------------------+-------------+------------------+-----------+-------------+----------------+ + | ``--strict-sandbox`` | Yes | Stable | Yes | Yes | Stable | + +----------------------+-------------+------------------+-----------+-------------+----------------+ + + The overall behaviour depends on the availability of a sandbox image. Such + an image must be provided by a recipe via + :ref:`configuration-recipes-provideSandbox` and the sandbox image must have + been picked up by a ``use: [sandbox]`` dependency. + + The execution path is the path where the checkout/build/packageScript is + executed. This is usually the *workspace* path but some modes use a + *stable* path instead. Stable paths start with ``/bob/...`` and are computed + from the :term:`Variant-Id` of the step. An unchanged step will always be + executed at the same stable path in a sandbox. + +Using ``--no-sandbox`` will not use any sandboxing features and all build steps +are executed without any isolation on the build host. The ``--sandbox`` option +will provide partial isolation only if a sandbox image is available for a package. +Inside the sandbox image all paths are stable, i.e. independent of the +workspace path. As a light-weight alternative, the ``--slim-sandbox`` option +will always provide isolation but an available sandbox image is not used and +all workspace paths are retained. Likewise, the ``--dev-sandbox`` option will +also provide full isolation but an available sandbox image is used. The +``--strict-sandbox`` option further uses stable paths consistently. + + Options ------- @@ -65,6 +126,13 @@ Options enable ``--with-provided`` to build and copy all provided packages of the built package(s). +``--dev-sandbox`` + Enable development sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. + ``--download MODE`` Download from binary archive (yes, no, deps, forced, forced-deps, packages) @@ -134,11 +202,31 @@ Options failed and the error has been corrected in the failing package. ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + + Build packages in an ephemeral container if a sandbox image is available + for the package. Inside the sandbox, stable execution paths are used. In + absence of a sandbox image, no isolation is performed. ``--shared`` Use shared packages if they are available. This is the default. +``--slim-sandbox`` + Enable slim sandboxing. + + Build packages in an isolated mount namespace. Most of the host paths + are available read-only. Other workspaces are hidden when building a + package unless they are a declared dependency. An optionally available + sandbox image is *not* used. + +``--strict-sandbox`` + Enable strict sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. The build path is always + a reproducible, stable path. + ``--upload`` Upload to binary archive diff --git a/doc/manpages/bob-build.rst b/doc/manpages/bob-build.rst index e2142d9b..115a381f 100644 --- a/doc/manpages/bob-build.rst +++ b/doc/manpages/bob-build.rst @@ -22,7 +22,8 @@ Synopsis [-lc LAYERCONFIG] [-e NAME] [-E] [-M META] [--upload] [--link-deps] [--no-link-deps] [--download MODE] [--download-layer MODE] [--shared | --no-shared] - [--install | --no-install] [--sandbox | --no-sandbox] + [--install | --no-install] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] PACKAGE [PACKAGE ...] diff --git a/doc/manpages/bob-dev.rst b/doc/manpages/bob-dev.rst index f24b2724..3cdd25b1 100644 --- a/doc/manpages/bob-dev.rst +++ b/doc/manpages/bob-dev.rst @@ -22,11 +22,11 @@ Synopsis [-lc LAYERCONFIG] [-e NAME] [-E] [-M META] [--upload] [--link-deps] [--no-link-deps] [--download MODE] [--download-layer MODE] [--shared | --no-shared] - [--install | --no-install] [--sandbox | --no-sandbox] + [--install | --no-install] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] PACKAGE [PACKAGE ...] - Description ----------- diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index a717d108..587fd039 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -2298,7 +2298,10 @@ jobs ``-j`` Integer link_deps ``--[no-]link-deps`` Boolean no_deps ``-n`` Boolean no_logfiles ``--no-logfiles`` Boolean -sandbox ``--[no-]sandbox`` Boolean +sandbox ``--[no-]sandbox`` | Boolean / String (``yes``, ``no``, ``slim``, + ``--slim-sandbox`` | ``dev``, ``strict``) + ``--dev-sandbox`` | + ``--strict-sandbox`` shared ``--[no-]shared`` Boolean upload ``--upload`` Boolean verbosity ``-q | -v`` Integer (-2[quiet] .. 3[verbose], default 0) From 00a77e81482ab1dd769a6c3c37b2b5502df395d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 23:52:55 +0200 Subject: [PATCH 10/17] jenkins: add support for new sandbox modes Adds the new --slim-sandbox, --dev-sandbox and --strict-sandbox options to the jenkins commands. --- pym/bob/cmds/jenkins/exec.py | 4 ++ pym/bob/cmds/jenkins/jenkins.py | 55 +++++++++++++++++++++------ pym/bob/state.py | 19 +++++++-- test/unit/test_jenkins_set_options.py | 4 +- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/pym/bob/cmds/jenkins/exec.py b/pym/bob/cmds/jenkins/exec.py index 52ec7190..8c82d437 100644 --- a/pym/bob/cmds/jenkins/exec.py +++ b/pym/bob/cmds/jenkins/exec.py @@ -30,6 +30,7 @@ def __init__(self, specFile): self.platform = None self.__recipesAudit = [] self.__execIR = [] + self.slimSandbox = False with open(specFile, "r") as f: f.readline() # skip shebang @@ -65,6 +66,8 @@ def __handleCfg(self, line): self.platform = v elif k == "always-checkout": self.scmAlwaysCheckout = isTrue(v) + elif k == "slimSandbox": + self.slimSandbox = isTrue(v) else: raise AssertionError(line) @@ -237,6 +240,7 @@ def doJenkinsExecuteRun(argv, bobRoot): builder.setShareMode(True, True) if spec.scmAlwaysCheckout: builder.setAlwaysCheckout([".*"]) + builder.setSlimSandbox(spec.slimSandbox) builder.cook(ir.getRoots(), False, loop) return 0 diff --git a/pym/bob/cmds/jenkins/jenkins.py b/pym/bob/cmds/jenkins/jenkins.py index f3f08f60..3dc64f29 100644 --- a/pym/bob/cmds/jenkins/jenkins.py +++ b/pym/bob/cmds/jenkins/jenkins.py @@ -11,7 +11,7 @@ from ...state import BobState, JenkinsConfig from ...tty import WarnOnce from ...utils import processDefines, runInEventLoop, sslNoVerifyContext, quoteCmdExe, \ - getPlatformString + getPlatformString, SandboxMode, compareVersion from .intermediate import getJenkinsVariantId, PartialIR from pathlib import PurePosixPath from shlex import quote @@ -443,6 +443,7 @@ def dumpXML(self, orig, config, date, recipesAudit=None): "copy=" + config.artifactsCopy, "share=" + config.sharedDir, "always-checkout=" + ("1" if config.scmAlwaysCheckout else "0"), + "slimSandbox=" + ("1" if config.sandbox.slimSandbox else "0"), ]) if config.sharedQuota: execCmds.append("quota=" + config.sharedQuota) @@ -742,7 +743,8 @@ def genJenkinsJobs(recipes, jenkins): packages = recipes.generatePackages( jenkinsNamePersister(jenkins, nameFormatter, config.uuid), - config.sandbox) + config.sandbox.sandboxEnabled, + config.sandbox.stablePaths) nameCalculator = JobNameCalculator(config.prefix) rootPackages = [] for r in config.roots: rootPackages.extend(packages.queryPackagePath(r)) @@ -807,7 +809,16 @@ def doJenkinsAdd(recipes, argv): help="Download from binary archive") parser.add_argument('--upload', default=False, action='store_true', help="Upload to binary archive") - parser.add_argument('--no-sandbox', action='store_false', dest='sandbox', default=True, + group = parser.add_mutually_exclusive_group() + group.add_argument('--sandbox', action='store_const', const="yes", default="yes", + help="Enable partial sandboxing") + group.add_argument('--slim-sandbox', action='store_const', const="slim", dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_const', const="dev", dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_const', const="strict", dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_const', const="no", dest='sandbox', help="Disable sandboxing") parser.add_argument("--credentials", help="Credentials UUID for SCM checkouts") group = parser.add_mutually_exclusive_group() @@ -836,7 +847,7 @@ def doJenkinsAdd(recipes, argv): config.defines = processDefines(args.defines) config.download = args.download config.upload = args.upload - config.sandbox = args.sandbox + config.sandbox = SandboxMode(args.sandbox) config.credentials = args.credentials config.clean = args.clean config.keep = args.keep @@ -884,7 +895,7 @@ def doJenkinsExport(recipes, argv): 'url' : config.url, 'prefix' : config.prefix, 'nodes' : config.nodes, - 'sandbox' : config.sandbox, + 'sandbox' : config.sandbox.mode, 'windows' : config.windows, 'hostPlatform' : config.hostPlatform, 'checkoutSteps' : job.getCheckoutSteps(), @@ -917,6 +928,14 @@ def doJenkinsLs(recipes, argv): help="Show additional information") args = parser.parse_args(argv) + SANDBOX_MODES = { + "no" : "disabled", + "yes" : "enabled, partial", + "slim" : "enabled, slim", + "dev" : "enabled, dev", + "strict" : "enabled, strict", + } + for j in sorted(BobState().getAllJenkins()): print(j) cfg = BobState().getJenkinsConfig(j) @@ -933,7 +952,7 @@ def doJenkinsLs(recipes, argv): print(" Download:", "enabled" if cfg.download else "disabled") print(" Upload:", "enabled" if cfg.upload else "disabled") print(" Clean builds:", "enabled" if cfg.clean else "disabled") - print(" Sandbox:", "enabled" if cfg.sandbox else "disabled") + print(" Sandbox:", SANDBOX_MODES[cfg.sandbox.mode]) if cfg.credentials: print(" Credentials:", cfg.credentials) options = cfg.getOptions() @@ -1207,7 +1226,13 @@ def doJenkinsRm(recipes, argv): def applyHooks(hooks, job, info, reverse=False): for hook, apiVersion in (reversed(hooks) if reverse else hooks): - job = hook(job, **info) + if compareVersion(apiVersion, "0.24.1.dev106") < 0: + # Retain pre 0.25 compatibility of 'sandbox' info item + args = info.copy() + args['sandbox'] = args['sandbox'] != "no" + else: + args = info + job = hook(job, **args) return job def doJenkinsPush(recipes, argv): @@ -1288,7 +1313,7 @@ def printDebug(job, *args): printLine(2, job, *args) 'url' : config.url, 'prefix' : config.prefix, 'nodes' : config.nodes, - 'sandbox' : config.sandbox, + 'sandbox' : config.sandbox.mode, 'windows' : config.windows, 'hostPlatform' : config.hostPlatform, 'checkoutSteps' : job.getCheckoutSteps(), @@ -1501,9 +1526,15 @@ def doJenkinsSetOptions(recipes, argv): group.add_argument('--no-upload', action='store_false', dest='upload', help="Disable binary archive upload") group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=None, - help="Enable sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', + group.add_argument('--sandbox', action='store_const', const="yes", default=None, + help="Enable partial sandboxing") + group.add_argument('--slim-sandbox', action='store_const', const="slim", dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_const', const="dev", dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_const', const="strict", dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_const', const="no", dest='sandbox', help="Disable sandboxing") group = parser.add_mutually_exclusive_group() group.add_argument('--clean', action='store_true', default=None, @@ -1543,7 +1574,7 @@ def doJenkinsSetOptions(recipes, argv): if args.upload is not None: config.upload = args.upload if args.sandbox is not None: - config.sandbox = args.sandbox + config.sandbox = SandboxMode(args.sandbox) if defines: config.defines.update(defines) for d in args.undefines: diff --git a/pym/bob/state.py b/pym/bob/state.py index daa912ca..45454fff 100644 --- a/pym/bob/state.py +++ b/pym/bob/state.py @@ -6,7 +6,7 @@ from .errors import ParseError from .stringparser import isTrue from .tty import colorize, WarnOnce, WARNING -from .utils import replacePath, getPlatformString +from .utils import replacePath, getPlatformString, SandboxMode import copy import errno import os @@ -52,7 +52,7 @@ def load(cls, config): self.defines = config.get("defines", {}).copy() self.download = config.get("download", False) self.upload = config.get("upload", False) - self.sandbox = config.get("sandbox", True) + self.__sandbox = SandboxMode(config.get("sandbox", True)) self.credentials = config.get("credentials", None) self.clean = config.get("clean", True) self.keep = config.get("keep", False) @@ -83,7 +83,7 @@ def dump(self): "options" : self.__options, "prefix" : self.prefix, "roots" : self.roots, - "sandbox" : self.sandbox, + "sandbox" : self.__sandbox.compatMode, "shortdescription" : self.shortdescription, "upload" : self.upload, "url" : self.__url, @@ -97,7 +97,7 @@ def reset(self): self.defines = {} self.download = False self.upload = False - self.sandbox = True + self.__sandbox = SandboxMode(True) self.credentials = None self.clean = True self.keep = False @@ -106,6 +106,17 @@ def reset(self): self.hostPlatform = getPlatformString() self.__options = {} + @property + def sandbox(self): + return self.__sandbox + + @sandbox.setter + def sandbox(self, mode): + if isinstance(mode, SandboxMode): + self.__sandbox = mode + else: + self.__sandbox = SandboxMode(mode) + @property def url(self): url = self.__url diff --git a/test/unit/test_jenkins_set_options.py b/test/unit/test_jenkins_set_options.py index 37232d24..61cde685 100644 --- a/test/unit/test_jenkins_set_options.py +++ b/test/unit/test_jenkins_set_options.py @@ -66,7 +66,7 @@ def testAllOptions(self): self.assertTrue(c.keep) self.assertTrue(c.download) self.assertTrue(c.upload) - self.assertFalse(c.sandbox) + self.assertEqual(c.sandbox.mode, "no") self.assertFalse(c.clean) def testReset(self): @@ -84,5 +84,5 @@ def testReset(self): self.assertFalse(c.keep) self.assertFalse(c.download) self.assertFalse(c.upload) - self.assertTrue(c.sandbox) + self.assertEqual(c.sandbox.mode, "yes") self.assertTrue(c.clean) From e039dcf6a9749cd94c8a22017602193d958f7f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 23:54:37 +0200 Subject: [PATCH 11/17] doc: document new jenkins sandbox modes --- doc/manpages/bob-jenkins.rst | 38 +++++++++++++++++++++++++++++++----- doc/manual/extending.rst | 4 +++- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/doc/manpages/bob-jenkins.rst b/doc/manpages/bob-jenkins.rst index 7cc9c366..e82a897e 100644 --- a/doc/manpages/bob-jenkins.rst +++ b/doc/manpages/bob-jenkins.rst @@ -26,8 +26,8 @@ Available sub-commands: bob jenkins add [-h] [-n NODES] [-o OPTIONS] [--host-platform {linux,msys,win32}] [-w] [-p PREFIX] [-r ROOT] [-D DEFINES] [--keep] [--download] [--upload] - [--no-sandbox] [--credentials CREDENTIALS] - [--clean | --incremental] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] + [--credentials CREDENTIALS] [--clean | --incremental] [--shortdescription | --longdescription] name url bob jenkins export [-h] name dir @@ -50,7 +50,7 @@ Available sub-commands: [--keep | --no-keep] [--download | --no-download] [--upload | --no-upload] - [--sandbox | --no-sandbox] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean | --incremental] name bob jenkins set-url [-h] name url @@ -126,6 +126,13 @@ Options ``--del-root DEL_ROOT`` Remove existing root package. +``--dev-sandbox`` + Enable development sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. + ``--download`` Enable downloads from binary archive. Disabled by default. There must be at least one binary archive in the user configuration @@ -224,7 +231,8 @@ Options Disable sandboxing during builds. Unless required by the project, it is discouraged to disable the sandbox - feature. See ``--sandbox`` for the opposite switch. + feature. See ``--sandbox``, ``--slim-sandbox``, ``--dev-sandbox`` or + ``--strict-sandbox`` for the opposite switches. ``--no-ssl-verify`` Disable HTTPS certificate checking. @@ -296,7 +304,11 @@ Options on the previous state. ``--sandbox`` - Enable sandboxing. This is the default. + Enable partial sandboxing. This is the default. + + Build packages in an ephemeral container if a sandbox image is available + for the package. Inside the sandbox, stable execution paths are used. In + absence of a sandbox image, no isolation is performed. ``--shortdescription`` Do not calculate every possible path of each package in a job for the @@ -305,6 +317,22 @@ Options drawback is that not all packages are then listed in the job description. For each unique package only one example path will be shown. +``--slim-sandbox`` + Enable slim sandboxing. + + Build packages in an isolated mount namespace. Most of the host paths + are available read-only. Other workspaces are hidden when building a + package unless they are a declared dependency. An optionally available + sandbox image is *not* used. + +``--strict-sandbox`` + Enable strict sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. The build path is always + a reproducible, stable path. + ``-U UNDEFINES`` Undefine environment variable override. This removes a variable previously defined with ``-D``. diff --git a/doc/manual/extending.rst b/doc/manual/extending.rst index bd22f846..ea64c44c 100644 --- a/doc/manual/extending.rst +++ b/doc/manual/extending.rst @@ -209,7 +209,9 @@ Currently the following additional kwargs are passed: * ``packageSteps``: list of all package steps (:class:`bob.input.Step`) used in the job * ``prefix``: Prefix of all job names - * ``sandbox``: Boolean whether sandbox should be used + * ``sandbox``: Boolean whether sandbox should be used. Plugins starting with + API version ``0.25`` are passed the sandbox mode as string (``no``, ``yes``, + ``slim``, ``dev`` or ``strict``). * ``url``: URL of Jenkins instance * ``windows``: True if Jenkins runs on Windows From a41b7074e2e0cc181592a2602508144685bfee76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Thu, 10 Oct 2024 23:55:33 +0200 Subject: [PATCH 12/17] test: add tests for new sandbox modes --- test/black-box/sandbox/config.yaml | 1 + test/black-box/sandbox/recipes/isolation.yaml | 11 ++ test/black-box/sandbox/recipes/sandbox.yaml | 3 + test/black-box/sandbox/recipes/test.yaml | 61 +++++++ test/black-box/sandbox/run.sh | 31 ++++ test/unit/test_jenkins_build.py | 154 +++++++++++++++++- test/unit/test_jenkins_set_options.py | 12 ++ 7 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 test/black-box/sandbox/config.yaml create mode 100644 test/black-box/sandbox/recipes/isolation.yaml create mode 100644 test/black-box/sandbox/recipes/test.yaml diff --git a/test/black-box/sandbox/config.yaml b/test/black-box/sandbox/config.yaml new file mode 100644 index 00000000..e97967d5 --- /dev/null +++ b/test/black-box/sandbox/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: "0.24" diff --git a/test/black-box/sandbox/recipes/isolation.yaml b/test/black-box/sandbox/recipes/isolation.yaml new file mode 100644 index 00000000..bffb3563 --- /dev/null +++ b/test/black-box/sandbox/recipes/isolation.yaml @@ -0,0 +1,11 @@ +root: True + +depends: + - test-outside + - name: sandbox + use: [sandbox] + forward: True + - test-inside + +buildScript: "true" +packageScript: "true" diff --git a/test/black-box/sandbox/recipes/sandbox.yaml b/test/black-box/sandbox/recipes/sandbox.yaml index 68fef551..1fce162c 100644 --- a/test/black-box/sandbox/recipes/sandbox.yaml +++ b/test/black-box/sandbox/recipes/sandbox.yaml @@ -1,4 +1,7 @@ # Empty sandbox that mounts the whole host +packageScript: | + echo "canary" > canary.txt + provideSandbox: paths: ["/usr/local/bin", "/usr/local/sbin", "/usr/bin", "/usr/sbin", "/bin", "/sbin"] diff --git a/test/black-box/sandbox/recipes/test.yaml b/test/black-box/sandbox/recipes/test.yaml new file mode 100644 index 00000000..61f93715 --- /dev/null +++ b/test/black-box/sandbox/recipes/test.yaml @@ -0,0 +1,61 @@ +packageVars: [CANARY] +packageScript: | + verifyIsolated() + { + if [[ $1 -ne 0 ]] ; then + if [[ -e "$CANARY" ]] ; then + echo "$CANARY exists in isolated environment" >&2 + exit 1 + fi + else + if [[ ! -e "$CANARY" ]] ; then + echo "$CANARY does not exists in host environment" >&2 + exit 1 + fi + fi + } + + verifyImageUsed() + { + if [[ $1 -ne 0 ]] ; then + if [[ ! -e /canary.txt ]] ; then + echo "Sandbox image not used?" >&2 + exit 1 + fi + else + if [[ -e /canary.txt ]] ; then + echo "Canary found in host environment" >&2 + exit 1 + fi + fi + } + + verifyStablePath() + { + if [[ $1 -ne 0 ]] ; then + if [[ $PWD != /bob/* ]] ; then + echo "No stable path inside sandbox" >&2 + exit 1 + fi + else + if [[ $PWD == /bob/* ]] ; then + echo "Stable path used in host environment" >&2 + exit 1 + fi + fi + } + +multiPackage: + outside: + packageVars: [OUTSIDE_ISOLATED, OUTSIDE_STABLE_PATH] + packageScript: | + verifyIsolated $OUTSIDE_ISOLATED + verifyImageUsed 0 + verifyStablePath $OUTSIDE_STABLE_PATH + + inside: + packageVars: [INSIDE_ISOLATED, INSIDE_IMAGE_USED, INSIDE_STABLE_PATH] + packageScript: | + verifyIsolated $INSIDE_ISOLATED + verifyImageUsed $INSIDE_IMAGE_USED + verifyStablePath $INSIDE_STABLE_PATH diff --git a/test/black-box/sandbox/run.sh b/test/black-box/sandbox/run.sh index fce93f0c..7f4eb90b 100755 --- a/test/black-box/sandbox/run.sh +++ b/test/black-box/sandbox/run.sh @@ -14,3 +14,34 @@ run_bob dev --sandbox -E root # Check that we can keep our UID or even be root inside sandbox run_bob dev --sandbox -c as-self -DEXPECT_UID=$UID root run_bob dev --sandbox -c as-root -DEXPECT_UID=0 root + +# Test properties in- and outside of sandbox image with respect to the various +# modes... + +# The plain "--sandbox" mode executes packages without sandbox image unprotected. +# With an image, the step is executed in a container with stable paths. +cleanup +run_bob dev isolation -D CANARY="$PWD/run.sh" --sandbox \ + -D OUTSIDE_ISOLATED=0 -D OUTSIDE_STABLE_PATH=0 \ + -D INSIDE_ISOLATED=1 -D INSIDE_STABLE_PATH=1 -D INSIDE_IMAGE_USED=1 + +# The slim sandbox mode does not use the sandbox image. It will always use the +# workspace paths but will execute still in an isolated environment. +cleanup +run_bob dev isolation -D CANARY="$PWD/run.sh" --slim-sandbox \ + -D OUTSIDE_ISOLATED=1 -D OUTSIDE_STABLE_PATH=0 \ + -D INSIDE_ISOLATED=1 -D INSIDE_STABLE_PATH=0 -D INSIDE_IMAGE_USED=0 + +# The dev sandbox mode is like the slim mode, but will use the sandbox image if +# available... +cleanup +run_bob dev isolation -D CANARY="$PWD/run.sh" --dev-sandbox \ + -D OUTSIDE_ISOLATED=1 -D OUTSIDE_STABLE_PATH=0 \ + -D INSIDE_ISOLATED=1 -D INSIDE_STABLE_PATH=0 -D INSIDE_IMAGE_USED=1 + +# The strict sandbox mode always executes the steps in isolation with stable +# paths. If a sandbox image is available, it is used. +cleanup +run_bob dev isolation -D CANARY="$PWD/run.sh" --strict-sandbox \ + -D OUTSIDE_ISOLATED=1 -D OUTSIDE_STABLE_PATH=1 \ + -D INSIDE_ISOLATED=1 -D INSIDE_STABLE_PATH=1 -D INSIDE_IMAGE_USED=1 diff --git a/test/unit/test_jenkins_build.py b/test/unit/test_jenkins_build.py index 0ba8dd0a..63204a6e 100644 --- a/test/unit/test_jenkins_build.py +++ b/test/unit/test_jenkins_build.py @@ -1,9 +1,10 @@ from mocks.jenkins_tests import JenkinsTests from shlex import quote -from unittest import TestCase, expectedFailure +from unittest import TestCase, expectedFailure, skipUnless import os, os.path import tempfile import subprocess +import sys from bob.utils import removePath @@ -357,8 +358,159 @@ def testCustomSharedLocation(self): self.assertTrue(os.path.isdir(s)) # Build with tool + # Build with sandbox +class JenkinsSandboxBuilds(JenkinsTests): + OPTIONS = "--no-sandbox" + OUTSIDE_ISOLATED = 0 + OUTSIDE_STABLE_PATH = 0 + INSIDE_ISOLATED = 0 + INSIDE_IMAGE_USED = 0 + INSIDE_STABLE_PATH = 0 + + def testBuild(self): + """Test sandbox build""" + self.writeRecipe("root", """\ + root: True + depends: + - test-outside + - name: sandbox + use: [sandbox] + forward: True + - test-inside + + buildScript: "true" + packageScript: | + echo ok > result.txt + """) + self.writeRecipe("sandbox", """\ + packageScript: | + echo "canary" > canary.txt + provideSandbox: + paths: ["/usr/local/bin", "/usr/local/sbin", "/usr/bin", "/usr/sbin", + "/bin", "/sbin"] + mount: + - /bin + - /etc + - /lib + - /run + - /usr + - /var + - ["/lib32", "/lib32", [nofail]] + - ["/lib64", "/lib64", [nofail]] + """) + + # The canary needs to be somewhere in the project path. The slim sandbox + # mode only restricts those paths... + CANARY = os.path.join(self.cwd, "config.yaml") + self.writeRecipe("test", f"""\ + packageScript: | + verifyIsolated() + {{ + if [[ $1 -ne 0 ]] ; then + if [[ -e "{CANARY}" ]] ; then + echo "{CANARY} exists in isolated environment" >&2 + exit 1 + fi + else + if [[ ! -e "{CANARY}" ]] ; then + echo "{CANARY} does not exists in host environment" >&2 + exit 1 + fi + fi + }} + + verifyImageUsed() + {{ + if [[ $1 -ne 0 ]] ; then + if [[ ! -e /canary.txt ]] ; then + echo "Sandbox image not used?" >&2 + exit 1 + fi + else + if [[ -e /canary.txt ]] ; then + echo "Canary found in host environment" >&2 + exit 1 + fi + fi + }} + + verifyStablePath() + {{ + if [[ $1 -ne 0 ]] ; then + if [[ $PWD != /bob/* ]] ; then + echo "No stable path inside sandbox" >&2 + exit 1 + fi + else + if [[ $PWD == /bob/* ]] ; then + echo "Stable path used in host environment" >&2 + exit 1 + fi + fi + }} + + multiPackage: + outside: + packageScript: | + verifyIsolated {self.OUTSIDE_ISOLATED} + verifyImageUsed 0 + verifyStablePath {self.OUTSIDE_STABLE_PATH} + + inside: + packageScript: | + verifyIsolated {self.INSIDE_ISOLATED} + verifyImageUsed {self.INSIDE_IMAGE_USED} + verifyStablePath {self.INSIDE_STABLE_PATH} + """) + self.executeBobJenkinsCmd("set-options test " + self.OPTIONS) + self.executeBobJenkinsCmd("push test") + self.jenkinsMock.run() + with self.getJobResult("root") as d: + with open(os.path.join(d, "result.txt")) as f: + self.assertEqual(f.read(), "ok\n") + +@skipUnless(sys.platform == "linux", "Sandbox requires Linux") +class JenkinsSandboxBuildDisabled(JenkinsSandboxBuilds, TestCase): + pass + +@skipUnless(sys.platform == "linux", "Sandbox requires Linux") +class JenkinsSandboxBuildPartial(JenkinsSandboxBuilds, TestCase): + OPTIONS = "--sandbox" + OUTSIDE_ISOLATED=0 + OUTSIDE_STABLE_PATH=0 + INSIDE_ISOLATED=1 + INSIDE_STABLE_PATH=1 + INSIDE_IMAGE_USED=1 + +@skipUnless(sys.platform == "linux", "Sandbox requires Linux") +class JenkinsSandboxBuildSlim(JenkinsSandboxBuilds, TestCase): + OPTIONS = "--slim-sandbox" + OUTSIDE_ISOLATED=1 + OUTSIDE_STABLE_PATH=0 + INSIDE_ISOLATED=1 + INSIDE_STABLE_PATH=0 + INSIDE_IMAGE_USED=0 + +@skipUnless(sys.platform == "linux", "Sandbox requires Linux") +class JenkinsSandboxBuildDev(JenkinsSandboxBuilds, TestCase): + OPTIONS = "--dev-sandbox" + OUTSIDE_ISOLATED=1 + OUTSIDE_STABLE_PATH=0 + INSIDE_ISOLATED=1 + INSIDE_STABLE_PATH=0 + INSIDE_IMAGE_USED=1 + +@skipUnless(sys.platform == "linux", "Sandbox requires Linux") +class JenkinsSandboxBuildStrict(JenkinsSandboxBuilds, TestCase): + OPTIONS = "--strict-sandbox" + OUTSIDE_ISOLATED=1 + OUTSIDE_STABLE_PATH=1 + INSIDE_ISOLATED=1 + INSIDE_STABLE_PATH=1 + INSIDE_IMAGE_USED=1 + # Smoke tests: # - Set a node # - Set jobs.policy diff --git a/test/unit/test_jenkins_set_options.py b/test/unit/test_jenkins_set_options.py index 61cde685..29f00c2b 100644 --- a/test/unit/test_jenkins_set_options.py +++ b/test/unit/test_jenkins_set_options.py @@ -86,3 +86,15 @@ def testReset(self): self.assertFalse(c.upload) self.assertEqual(c.sandbox.mode, "yes") self.assertTrue(c.clean) + + def testSandboxModes(self): + self.executeBobJenkinsCmd("set-options test --no-sandbox") + self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "no") + self.executeBobJenkinsCmd("set-options test --sandbox") + self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "yes") + self.executeBobJenkinsCmd("set-options test --slim-sandbox") + self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "slim") + self.executeBobJenkinsCmd("set-options test --dev-sandbox") + self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "dev") + self.executeBobJenkinsCmd("set-options test --strict-sandbox") + self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "strict") From 948386d930178eea7f093c0816c313a43a1eae33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Oct 2024 14:34:20 +0200 Subject: [PATCH 13/17] doc: clarify is-sandbox-enabled function semantics The function actually tests the usage of a sandbox image. This is an important distinction because it won't return true in case of the slim sandbox build mode. Likewise, it will return false in the dev- or strict-sandbox modes when packages have no available sandbox image even though isolation is applied. --- doc/manual/configuration.rst | 2 +- doc/manual/extending.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 587fd039..e4e2d336 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -365,7 +365,7 @@ The following built in string functions are supported: * ``$(if-then-else,condition,then,else)``: The expansion of ``condition`` is interpreted as a boolean value. If the condition is true the expansion of ``then`` is returned. Otherwise ``else`` is returned. -* ``$(is-sandbox-enabled)``: Return ``true`` if a sandbox is enabled in the +* ``$(is-sandbox-enabled)``: Return ``true`` if a sandbox image is used in the current context, ``false`` otherwise. * ``$(is-tool-defined,name)``: If ``name`` is a defined tool in the current context the function will return ``true``. Otherwise ``false`` is returned. diff --git a/doc/manual/extending.rst b/doc/manual/extending.rst index ea64c44c..af7911b6 100644 --- a/doc/manual/extending.rst +++ b/doc/manual/extending.rst @@ -162,7 +162,7 @@ passed: * ``env``: dict of all available environment variables at the current context * ``recipe``: the current :class:`bob.input.Recipe` -* ``sandbox``: ``True`` if a sandbox is used. ``False`` if no sandbox was +* ``sandbox``: ``True`` if a sandbox *image* is used. ``False`` if no sandbox image was configured or if it is disabled (e.g. ``--no-sandbox`` option was specified). In the future additional keyword args may be added without notice. Such string From cec9a459318272b8c27805439851c3aaaa4920ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Oct 2024 16:24:34 +0200 Subject: [PATCH 14/17] test: fix query-path bob invocations The test incorrectly uses `--dev' as abbreviation for the '--develop' option. While this worked so far, it will be ambiguous with the introduction of the '--dev-sandbox' option. --- test/black-box/query-path/run.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/black-box/query-path/run.sh b/test/black-box/query-path/run.sh index 6b80ceaf..a4b7db17 100755 --- a/test/black-box/query-path/run.sh +++ b/test/black-box/query-path/run.sh @@ -52,20 +52,20 @@ test -n "$(run_bob query-path --release root/interm2/child)" # # Must report no path for all usages because we have not built. -test -z "$(run_bob query-path --dev root)" -test -z "$(run_bob query-path --dev root/interm1/child)" -test -z "$(run_bob query-path --dev root/interm2/child)" +test -z "$(run_bob query-path --develop root)" +test -z "$(run_bob query-path --develop root/interm1/child)" +test -z "$(run_bob query-path --develop root/interm2/child)" # Build everything run_bob dev root # Convert reported paths to unix format. -set -- $(run_bob query-path --dev root | sed -e 's|\\|/|g') +set -- $(run_bob query-path --develop root | sed -e 's|\\|/|g') test "$1" = "root" test "$2" = "dev/dist/root/1/workspace" -# 'query-path --dev' is the default +# 'query-path --develop' is the default PACKAGES="root root/interm1/child root/interm2/child" -RESULT=$(run_bob query-path --dev $PACKAGES) +RESULT=$(run_bob query-path --develop $PACKAGES) test "$RESULT" = "$(run_bob query-path $PACKAGES)" # No matter in which order we build, we must get the same result. @@ -73,7 +73,7 @@ for i in $PACKAGES; do rm -rf work dev .bob-* run_bob dev $i run_bob dev root - test "$(run_bob query-path --dev $PACKAGES)" = "$RESULT" + test "$(run_bob query-path --develop $PACKAGES)" = "$RESULT" done From 2c1da0712af9882ec096a741f3db12fd8b360702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Oct 2024 15:20:10 +0200 Subject: [PATCH 15/17] cmds: add new sandbox modes to remaining commands This does not actually add any new behaviour. But it makes the remaining commands consistent with the build/dev/jenkins commands by accepting the same options. --- pym/bob/cmds/build/project.py | 27 +++++++++++++++++++++------ pym/bob/cmds/build/query.py | 12 ++++++++++-- pym/bob/cmds/graph.py | 6 ++++++ pym/bob/cmds/misc.py | 24 ++++++++++++++++++++++++ pym/bob/cmds/show.py | 6 ++++++ 5 files changed, 67 insertions(+), 8 deletions(-) diff --git a/pym/bob/cmds/build/project.py b/pym/bob/cmds/build/project.py index 284b71a0..6d2aadbf 100644 --- a/pym/bob/cmds/build/project.py +++ b/pym/bob/cmds/build/project.py @@ -8,7 +8,7 @@ from ...generators import generators as defaultGenerators from ...input import RecipeSet from ...tty import colorize -from ...utils import processDefines +from ...utils import processDefines, SandboxMode import argparse import os @@ -48,9 +48,15 @@ def _downloadArg(arg): parser.add_argument('-j', '--jobs', default=None, type=int, nargs='?', const=..., help="Specifies the number of jobs to run simultaneously.") group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', + group.add_argument('--sandbox', action='store_const', const="yes", default="no", + help="Enable partial sandboxing") + group.add_argument('--slim-sandbox', action='store_const', const="slim", dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_const', const="dev", dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_const', const="strict", dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_const', const="no", dest='sandbox', help="Disable sandboxing") args = parser.parse_args(argv) @@ -68,7 +74,9 @@ def _downloadArg(arg): nameFormatter = recipes.getHook('developNameFormatter') developPersister = DevelopDirOracle(nameFormatter, recipes.getHook('developNamePersister')) nameFormatter = LocalBuilder.makeRunnable(developPersister.getFormatter()) - packages = recipes.generatePackages(nameFormatter, args.sandbox) + sandboxMode = SandboxMode(args.sandbox) + packages = recipes.generatePackages(nameFormatter, sandboxMode.sandboxEnabled, + sandboxMode.stablePaths) developPersister.prime(packages) generators = defaultGenerators.copy() @@ -98,7 +106,14 @@ def _downloadArg(arg): extra.append('-e') extra.append(e) if args.preserve_env: extra.append('-E') - if args.sandbox: extra.append('--sandbox') + if args.sandbox == "yes": + extra.append('--sandbox') + elif args.sandbox == "slim": + extra.append('--slim-sandbox') + elif args.sandbox == "dev": + extra.append('--dev-sandbox') + elif args.sandbox == "strict": + extra.append('--strict-sandbox') if args.jobs is ...: # expand because we cannot control the argument order in the generator args.jobs = os.cpu_count() diff --git a/pym/bob/cmds/build/query.py b/pym/bob/cmds/build/query.py index ab85069b..b6d9108d 100644 --- a/pym/bob/cmds/build/query.py +++ b/pym/bob/cmds/build/query.py @@ -45,8 +45,16 @@ def doQueryPath(argv, bobRoot): help="Return a non-zero error code in case of errors") group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', help="Enable sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") + group.add_argument('--sandbox', action='store_true', default=False, + help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_false', dest='sandbox', + help="Disable sandboxing") parser.set_defaults(sandbox=None) group = parser.add_mutually_exclusive_group() diff --git a/pym/bob/cmds/graph.py b/pym/bob/cmds/graph.py index 19edc05a..b75cfc5e 100644 --- a/pym/bob/cmds/graph.py +++ b/pym/bob/cmds/graph.py @@ -513,6 +513,12 @@ def doGraph(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") parser.add_argument('--destination', metavar="DEST", diff --git a/pym/bob/cmds/misc.py b/pym/bob/cmds/misc.py index 5428a958..fe0bfa21 100644 --- a/pym/bob/cmds/misc.py +++ b/pym/bob/cmds/misc.py @@ -90,6 +90,12 @@ def doLS(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") args = parser.parse_args(argv) @@ -138,6 +144,12 @@ def doQueryMeta(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") args = parser.parse_args(argv) @@ -196,6 +208,12 @@ def doQuerySCM(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") @@ -258,6 +276,12 @@ def doQueryRecipe(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") diff --git a/pym/bob/cmds/show.py b/pym/bob/cmds/show.py index e68429c7..8cd9d0e0 100644 --- a/pym/bob/cmds/show.py +++ b/pym/bob/cmds/show.py @@ -252,6 +252,12 @@ def doShow(argv, bobRoot): group = parser.add_mutually_exclusive_group() group.add_argument('--sandbox', action='store_true', default=False, help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") group.add_argument('--no-sandbox', action='store_false', dest='sandbox', help="Disable sandboxing") From 2d35b1d3b1c20516ff7af27df7f41d8ffc2d2692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Oct 2024 15:22:41 +0200 Subject: [PATCH 16/17] doc: document added sandbox switches of remaining commands --- doc/manpages/bob-graph.rst | 14 ++++++++++-- doc/manpages/bob-ls.rst | 15 ++++++++++--- doc/manpages/bob-project.rst | 33 +++++++++++++++++++++++++--- doc/manpages/bob-query-meta.rst | 13 +++++++++-- doc/manpages/bob-query-path.rst | 36 ++++++++++++++++++++----------- doc/manpages/bob-query-recipe.rst | 13 +++++++++-- doc/manpages/bob-query-scm.rst | 15 +++++++++++-- doc/manpages/bob-show.rst | 14 ++++++++++-- 8 files changed, 124 insertions(+), 29 deletions(-) diff --git a/doc/manpages/bob-graph.rst b/doc/manpages/bob-graph.rst index 221b825f..50aad792 100644 --- a/doc/manpages/bob-graph.rst +++ b/doc/manpages/bob-graph.rst @@ -15,7 +15,8 @@ Synopsis :: - bob graph [-h] [-D DEFINES] [-c CONFIGFILE] [--sandbox | --no-sandbox] + bob graph [-h] [-D DEFINES] [-c CONFIGFILE] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--destination DEST] [-e EXCLUDES] [-f FILENAME] [-H HIGHLIGHTS] [-n MAX_DEPTH] [-t {d3,dot}] [-o OPTIONS] PACKAGE [PACKAGE ...] @@ -45,6 +46,9 @@ Options ``--destination`` Destination of graph output files. +``--dev-sandbox`` + Enable development sandboxing. Include sandbox dependencies in the graph. + ``-e, --excludes`` Do not show packages matching this regex. (And all it's dependencies) @@ -64,7 +68,13 @@ Options the default. ``--sandbox`` - Enable sandboxing. Include sandbox dependencies in the graph. + Enable partial sandboxing. Include sandbox dependencies in the graph. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. Include sandbox dependencies in the graph. ``-t, --type`` Set the graph type. ``d3`` (default) or ``dot``. diff --git a/doc/manpages/bob-ls.rst b/doc/manpages/bob-ls.rst index fa229eb7..760a0c40 100644 --- a/doc/manpages/bob-ls.rst +++ b/doc/manpages/bob-ls.rst @@ -16,10 +16,10 @@ Synopsis :: bob ls [-h] [-a] [-A] [-o] [-r] [-u] [-p | -d] [-D DEFINES] - [-c CONFIGFILE] [--sandbox | --no-sandbox] + [-c CONFIGFILE] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [package] - Description ----------- @@ -80,6 +80,9 @@ Options ``-D DEFINES`` Override default environment variable +``--dev-sandbox`` + Enable development sandboxing. + ``--no-sandbox`` Disable sandboxing @@ -95,7 +98,13 @@ Options Recursively display dependencies ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. ``-u, --unsorted`` Show the packages in the order they were named in the recipe. By default diff --git a/doc/manpages/bob-project.rst b/doc/manpages/bob-project.rst index 76d29045..6dc616a3 100644 --- a/doc/manpages/bob-project.rst +++ b/doc/manpages/bob-project.rst @@ -17,7 +17,7 @@ Synopsis bob project [-h] [--list] [-D DEFINES] [-c CONFIGFILE] [-e NAME] [-E] [--download MODE] [--resume] [-n] [-b] [-j [JOBS]] - [--sandbox | --no-sandbox] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [projectGenerator] [package] ... @@ -32,6 +32,13 @@ Options ``-c CONFIGFILE`` Use config File +``--dev-sandbox`` + Enable development sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. + ``--download MODE`` Download from binary archive (yes, no, deps, packages) @@ -57,7 +64,7 @@ Options work ``--no-sandbox`` - Disable sandboxing + Disable sandboxing. This is the default. ``-b`` Do build only (bob dev -b) before generate project Files. No checkout @@ -66,7 +73,27 @@ Options Resume build where it was previously interrupted ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + + Build packages in an ephemeral container if a sandbox image is available + for the package. Inside the sandbox, stable execution paths are used. In + absence of a sandbox image, no isolation is performed. + +``--slim-sandbox`` + Enable slim sandboxing. + + Build packages in an isolated mount namespace. Most of the host paths + are available read-only. Other workspaces are hidden when building a + package unless they are a declared dependency. An optionally available + sandbox image is *not* used. + +``--strict-sandbox`` + Enable strict sandboxing. + + Always build packages in an isolated environment where only declared + dependencies are visible. If a sandbox image is available, it is used. + Otherwise the host paths are made read-only. The build path is always + a reproducible, stable path. Eclipse CDT project generator ----------------------------- diff --git a/doc/manpages/bob-query-meta.rst b/doc/manpages/bob-query-meta.rst index 722d2546..aee1af7d 100644 --- a/doc/manpages/bob-query-meta.rst +++ b/doc/manpages/bob-query-meta.rst @@ -17,7 +17,7 @@ Synopsis :: bob query-meta [-h] [-D DEFINES] [-c CONFIGFILE] [-r] - [--sandbox | --no-sandbox] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] packages [packages ...] Description @@ -34,6 +34,9 @@ Options ``-D DEFINES`` Override default environment variable +``--dev-sandbox`` + Enable development sandboxing. + ``--no-sandbox`` Disable sandboxing @@ -41,4 +44,10 @@ Options Also list metaEnvironment variables for all dependencies. ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. diff --git a/doc/manpages/bob-query-path.rst b/doc/manpages/bob-query-path.rst index c93f3214..35df6001 100644 --- a/doc/manpages/bob-query-path.rst +++ b/doc/manpages/bob-query-path.rst @@ -15,9 +15,11 @@ Synopsis :: - bob query-path [-h] [-f FORMAT] [-D DEFINES] [-c CONFIGFILE] - [--sandbox | --no-sandbox] [--develop | --release] - [-q] [--fail] PACKAGE [PACKAGE ...] + bob query-path [-h] [-f FORMAT] [-D DEFINES] [-c CONFIGFILE] [-q] + [--fail] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] + [--develop | --release] + PACKAGE [PACKAGE ...] Description ----------- @@ -26,15 +28,15 @@ This command lists existing workspace directory names for packages given on the command line. Output is formatted with a format string that can contain placeholders - +----------+------------------+ - |{name} |package name | - +----------+------------------+ - |{src} |checkout directory| - +----------+------------------+ - |{build} |build directory | - +----------+------------------+ - |{dist} |package directory | - +----------+------------------+ + +-----------+--------------------+ + | {name} | package name | + +-----------+--------------------+ + | {src} | checkout directory | + +-----------+--------------------+ + | {build} | build directory | + +-----------+--------------------+ + | {dist} | package directory | + +-----------+--------------------+ The default format is '{name}{dist}'. @@ -52,6 +54,9 @@ Options ``-D DEFINES`` Override default environment variable +``--dev-sandbox`` + Enable development sandboxing. + ``--develop`` Use developer mode @@ -71,5 +76,10 @@ Options Use release mode ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + +``--slim-sandbox`` + Enable slim sandboxing. +``--strict-sandbox`` + Enable strict sandboxing. diff --git a/doc/manpages/bob-query-recipe.rst b/doc/manpages/bob-query-recipe.rst index d83b29a6..61557760 100644 --- a/doc/manpages/bob-query-recipe.rst +++ b/doc/manpages/bob-query-recipe.rst @@ -16,7 +16,7 @@ Synopsis :: bob query-recipe [-h] [-D DEFINES] [-c CONFIGFILE] - [--sandbox | --no-sandbox] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] package Description @@ -33,8 +33,17 @@ Options ``-D DEFINES`` Override default environment variable +``--dev-sandbox`` + Enable development sandboxing. + ``--no-sandbox`` Disable sandboxing ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. diff --git a/doc/manpages/bob-query-scm.rst b/doc/manpages/bob-query-scm.rst index 58e8bce4..e05b51d9 100644 --- a/doc/manpages/bob-query-scm.rst +++ b/doc/manpages/bob-query-scm.rst @@ -14,8 +14,10 @@ Synopsis -------- :: + bob query-scm [-h] [-D DEFINES] [-c CONFIGFILE] [-f FORMATS] - [--default DEFAULT] [-r] [--sandbox | --no-sandbox] + [--default DEFAULT] [-r] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] packages [packages ...] Description @@ -44,6 +46,9 @@ Options ``--default DEFAULT`` Default for missing attributes (default: "") +``--dev-sandbox`` + Enable development sandboxing. + ``-f FORMATS`` Output format for scm (syntax: scm=format). Can be specified multiple times. @@ -54,4 +59,10 @@ Options Recursively display dependencies ``--sandbox`` - Enable sandboxing + Enable partial sandboxing. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. diff --git a/doc/manpages/bob-show.rst b/doc/manpages/bob-show.rst index eca6ebef..1c656f31 100644 --- a/doc/manpages/bob-show.rst +++ b/doc/manpages/bob-show.rst @@ -15,7 +15,8 @@ Synopsis :: - bob show [-h] [-D DEFINES] [-c CONFIGFILE] [--sandbox | --no-sandbox] + bob show [-h] [-D DEFINES] [-c CONFIGFILE] + [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--show-empty] [--show-common] [--indent INDENT | --no-indent] [--format {yaml,json,flat,diff}] [-f {buildNetAccess,buildTools,buildToolsWeak,buildVars,buildVarsWeak, @@ -62,6 +63,9 @@ the properties that are different between both packages. Options ------- +``--dev-sandbox`` + Enable development sandboxing. + ``-f FIELD`` Show only the given ``FIELD``. This option may be specified more than once to show additional fields. If this option is not given then all fields are @@ -83,7 +87,7 @@ Options Disable sandboxing. ``--sandbox`` - Enable sandboxing. + Enable partial sandboxing. ``--show-common`` The ``diff`` format suppresses identical fields by default. This option @@ -94,3 +98,9 @@ Options as short as possible. Specifying this option will still show them. If the package does not have a checkout- or build-step the respective properties do not exist and will never be shown. + +``--slim-sandbox`` + Enable slim sandboxing. + +``--strict-sandbox`` + Enable strict sandboxing. From c51d5de85cc132d09e04561615898d19ada27117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Oct 2024 15:24:45 +0200 Subject: [PATCH 17/17] bash-completion: add new sandbox mode switches --- contrib/bash-completion/bob | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 3a2309a0..f48a4f0b 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -57,11 +57,8 @@ __bob_complete_path() # influence parsing and must be passed to "bob ls". for i in "${words[@]}"; do case "$i" in - --sandbox) - sandbox="--sandbox" - ;; - --no-sandbox) - sandbox="--no-sandbox" + --*sandbox) + sandbox="$i" ;; -c?* | -D?*) cmd_settings+=( "$i" ) @@ -130,7 +127,7 @@ __bob_cook() elif [[ "$prev" = "--always-checkout" ]] ; then COMPREPLY=( ) else - __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic" + __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean-checkout --attic --no-attic" fi } @@ -151,7 +148,7 @@ __bob_graph() if [[ "$prev" = "-t" || "$prev" = "--type" ]] ; then __bob_complete_words "d3 dot" else - __bob_complete_path "-c -D -e --exclude -f --filename -H --highlight -n --max-depth -t --type -o --sandbox --no-sandbox" + __bob_complete_path "-c -D -e --exclude -f --filename -H --highlight -n --max-depth -t --type -o --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox" fi } @@ -162,7 +159,7 @@ __bob_help() __bob_ls() { - __bob_complete_path "-a --all -c -D -d --direct --no-sandbox -o --origin -p --prefixed -r --recursive --sandbox -u --unsorted" + __bob_complete_path "-a --all -c -D -d --direct --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox -o --origin -p --prefixed -r --recursive --sandbox -u --unsorted" } __bob_init() @@ -183,7 +180,7 @@ __bob_jenkins_add() case "$cur" in -*) - __bob_complete_words "--clean --credentials --download --help --host-platform --keep --longdescription --no-sandbox --nodes --prefix --root --shortdescription --upload --windows -D -h -n -o -p -r -w" + __bob_complete_words "--clean --credentials --download --help --host-platform --keep --longdescription --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --nodes --prefix --root --shortdescription --upload --windows -D -h -n -o -p -r -w" ;; *) case "$prev" in @@ -304,7 +301,7 @@ __bob_jenkins_set_options() case "$cur" in -*) - __bob_complete_words "-h --help --reset -n --nodes -o -p --prefix --add-root --del-root -D -U --credentials --authtoken --shortdescription --longdescription --keep --no-keep --download --no-download --upload --no-upload --sandbox --no-sandbox --clean --incremental --host-platform" + __bob_complete_words "-h --help --reset -n --nodes -o -p --prefix --add-root --del-root -D -U --credentials --authtoken --shortdescription --longdescription --keep --no-keep --download --no-download --upload --no-upload --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean --incremental --host-platform" ;; *) case "$prev" in @@ -353,7 +350,7 @@ __bob_project() elif [[ -z "$command" ]] ; then case "$cur" in -*) - __bob_complete_words "-b -c -D --download -E -e -j --list -n --no-sandbox --resume --sandbox" + __bob_complete_words "-b -c -D --download -E -e -j --list -n --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --resume --sandbox" ;; *) __bob_complete_words "$($bob --color=never project --list 2>/dev/null)" @@ -366,22 +363,22 @@ __bob_project() __bob_query_scm() { - __bob_complete_path "-c -D -f --default -r --recursive --sandbox --no-sandbox" + __bob_complete_path "-c -D -f --default -r --recursive --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox" } __bob_query_path() { - __bob_complete_path "-f -D -c --sandbox --no-sandbox --develop --release" + __bob_complete_path "-f -D -c --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --develop --release" } __bob_query_meta() { - __bob_complete_path " -c -D -r --recursive --sandbox --no-sandbox" + __bob_complete_path " -c -D -r --recursive --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox" } __bob_query_recipe() { - __bob_complete_path "-c -D --sandbox --no-sandbox" + __bob_complete_path "-c -D --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox" } __bob_status() @@ -419,7 +416,8 @@ __bob_show() elif [[ "$prev" = "--indent" ]] ; then COMPREPLY=( ) else - __bob_complete_path "-D -c --sandbox --no-sandbox --show-empty + __bob_complete_path "-D -c --sandbox --no-sandbox --slim-sandbox + --dev-sandbox --strict-sandbox --show-empty --show-common --indent --no-indent --format -f" fi }