From 2b13716559080467d35de358f942273ef5bab475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 13 Jun 2021 22:08:49 +0200 Subject: [PATCH 1/2] languages: handle included files out of band Extracting included files in the executed script itself requires some code for each supported language. It's also cumbersome to clean up. Instead let the invoker handle the dirty details and just add the right reference in the final script. --- pym/bob/input.py | 47 ++++++++++++++++++---- pym/bob/invoker.py | 13 ++++++- pym/bob/languages.py | 93 +++++++++++++++++++------------------------- 3 files changed, 91 insertions(+), 62 deletions(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index 9281579f3..ed296a1e8 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -136,15 +136,19 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh): def mergeScripts(fragments, glue): """Join all scripts of the recipe and its classes. - The result is a tuple with (setupScript, mainScript, digestScript) + The result is a tuple with (setupScript, mainScript, digestScript, includedFiles) """ + return ( joinScripts((f[0][0] for f in fragments), glue), joinScripts((f[1][0] for f in fragments), glue), joinScripts( ( joinScripts((f[0][1] for f in fragments), "\n"), joinScripts((f[1][1] for f in fragments), "\n"), - ), "\n") + ), "\n"), + { name : content for name, content in + chain.from_iterable(chain(f[0][2].items(), f[1][2].items()) for f in fragments) + }, ) @@ -790,6 +794,9 @@ def getPostRunCmds(self): def getDigestScript(self): raise NotImplementedError + def getIncludedFiles(self): + raise NotImplementedError + def getLabel(self): raise NotImplementedError @@ -1042,6 +1049,9 @@ def getDigestScript(self): """ return self._coreStep.getDigestScript() + def getIncludedFiles(self): + return self._coreStep.getIncludedFiles() + def isDeterministic(self): """Return whether the step is deterministic. @@ -1458,6 +1468,9 @@ def getDigestScript(self): else: return None + def getIncludedFiles(self): + return self.corePackage.recipe.checkoutIncludedFiles + @property def fingerprintMask(self): return 0 @@ -1537,7 +1550,7 @@ def hasNetAccess(self): class CoreBuildStep(CoreStep): __slots__ = [] - def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]): + def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]): isValid = script[1] is not None super().__init__(corePackage, isValid, True, digestEnv, env, args) @@ -1568,6 +1581,9 @@ def getMainScript(self): def getDigestScript(self): return self.corePackage.recipe.buildDigestScript + def getIncludedFiles(self): + return self.corePackage.recipe.buildIncludedFiles + @property def fingerprintMask(self): # Remove bits of all tools that are not used in buildStep @@ -1591,7 +1607,7 @@ def hasNetAccess(self): class CorePackageStep(CoreStep): __slots__ = [] - def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]): + def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]): isValid = script[1] is not None super().__init__(corePackage, isValid, True, digestEnv, env, args) @@ -1622,6 +1638,9 @@ def getMainScript(self): def getDigestScript(self): return self.corePackage.recipe.packageDigestScript + def getIncludedFiles(self): + return self.corePackage.recipe.packageIncludedFiles + @property def fingerprintMask(self): return self.corePackage.fingerprintMask @@ -1884,7 +1903,7 @@ def resolve(self, text, section): raise ParseError("Bad substiturion in {}: {}".format(section, str(e))) return resolver.resolve(ret) else: - return (None, None) + return (None, None, {}) def mergeFilter(left, right): if left is None: @@ -2316,7 +2335,7 @@ def coDet(r): # the package step must always be valid if self.__package[1] is None: - self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709') + self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709', {}) # final shared value self.__shared = self.__shared == True @@ -2643,7 +2662,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, directPackages, indirectPackages, states, uidGen(), doFingerprint) # optional checkout step - if self.__checkout != (None, None, None) or self.__checkoutSCMs or self.__checkoutAsserts: + if self.__checkout != (None, None, None, {}) or self.__checkoutSCMs or self.__checkoutAsserts: checkoutDigestEnv = env.prune(self.__checkoutVars) checkoutEnv = ( env.prune(self.__checkoutVars | self.__checkoutVarsWeak) if self.__checkoutVarsWeak else checkoutDigestEnv ) @@ -2653,7 +2672,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, srcCoreStep = p.createInvalidCoreCheckoutStep() # optional build step - if self.__build != (None, None, None): + if self.__build != (None, None, None, {}): buildDigestEnv = env.prune(self.__buildVars) buildEnv = ( env.prune(self.__buildVars | self.__buildVarsWeak) if self.__buildVarsWeak else buildDigestEnv ) @@ -2765,6 +2784,10 @@ def checkoutMainScript(self): def checkoutDigestScript(self): return self.__checkout[2] or "" + @property + def checkoutIncludedFiles(self): + return self.__checkout[3] + @property def checkoutDeterministic(self): return self.__checkoutDeterministic @@ -2793,6 +2816,10 @@ def buildMainScript(self): def buildDigestScript(self): return self.__build[2] + @property + def buildIncludedFiles(self): + return self.__build[3] + @property def buildVars(self): return self.__buildVars @@ -2813,6 +2840,10 @@ def packageMainScript(self): def packageDigestScript(self): return self.__package[2] + @property + def packageIncludedFiles(self): + return self.__package[3] + @property def packageVars(self): return self.__packageVars diff --git a/pym/bob/invoker.py b/pym/bob/invoker.py index 3e56bfa9e..24888bb12 100644 --- a/pym/bob/invoker.py +++ b/pym/bob/invoker.py @@ -19,6 +19,7 @@ import os import re import shutil +import stat import subprocess import sys import tempfile @@ -266,7 +267,6 @@ def __getSandboxCmds(self, tmpDir): #FIXME: if verbosity >= 4: cmdArgs.append('-D') cmdArgs.extend(["-S", tmpDir]) cmdArgs.extend(["-H", "bob"]) - cmdArgs.extend(["-d", "/tmp"]) sandboxRootFs = os.path.abspath(self.__spec.sandboxRootWorkspace) for f in os.listdir(sandboxRootFs): cmdArgs.extend(["-M", os.path.join(sandboxRootFs, f), "-m", "/"+f]) @@ -301,6 +301,16 @@ async def executeStep(self, mode, clean=False, keepSandbox=False): # also used as ephemeral sandbox container. tmpDir = tempfile.mkdtemp() + # Create /tmp in sandbox container. Receives included files and + # needs to be there anyway. + os.mkdir(os.path.join(tmpDir, "tmp"), 0o755) + for name, content in self.__spec.includedFiles.items(): + fn = os.path.join(tmpDir, "tmp", name) + with open(fn, "wb") as f: + f.write(content) + # Set explicit mode to retain Bob 0.19 behaviour. + os.chmod(fn, stat.S_IREAD|stat.S_IWRITE) + # prepare workspace clean = self.__spec.clean if self.__spec.clean is not None else clean if not os.path.isdir(self.__spec.workspaceWorkspacePath): @@ -468,6 +478,7 @@ async def executeFingerprint(self, keepSandbox=False): # Setup workspace cmdArgs = self.__getSandboxCmds(tmpDir) + cmdArgs.extend(["-d", "/tmp"]) cmdArgs.extend(["-d", "/bob/fingerprint"]) cmdArgs.extend(["-W", "/bob/fingerprint"]) diff --git a/pym/bob/languages.py b/pym/bob/languages.py index efc89d9a2..58dadbdf2 100644 --- a/pym/bob/languages.py +++ b/pym/bob/languages.py @@ -6,8 +6,7 @@ from . import BOB_INPUT_HASH from .errors import ParseError from .utils import escapePwsh, quotePwsh, isWindows, asHexStr -from .utils import joinScripts, sliceString -from base64 import b64encode +from .utils import joinScripts, sliceString, INVALID_CHAR_TRANS from enum import Enum from glob import glob from shlex import quote @@ -148,7 +147,9 @@ def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): self.baseDir = baseDir self.sourceName = sourceName self.varBase = varBase + self.count = 0 self.__incDigests = [ hashlib.sha1(origText.encode('utf8')).digest().hex() ] + self.__incFiles = {} def __getitem__(self, item): mode = item[0] @@ -167,14 +168,17 @@ def __getitem__(self, item): self.__incDigests.append(hashlib.sha1(content).digest().hex()) if mode == '<': - ret = self._includeFile(content) + name = "{}_{}".format(self.varBase, self.count).translate(INVALID_CHAR_TRANS) + self.count += 1 + self.__incFiles[name] = content + ret = self._includeFile(name) else: assert mode == "'" ret = self._includeLiteral(content) return ret - def _includeFile(self, content): + def _includeFile(self, name): raise NotImplementedError() def _includeLiteral(self, content): @@ -184,7 +188,7 @@ def _resolveContent(self, result): raise NotImplementedError() def resolve(self, result): - return (self._resolveContent(result), "\n".join(self.__incDigests)) + return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles) # For each supported language the following runtime environments must be # considered: @@ -197,28 +201,15 @@ def resolve(self, result): # converted. class BashResolver(IncludeResolver): - def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): - super().__init__(fileLoader, baseDir, origText, sourceName, varBase) - self.prolog = [] - self.count = 0 - def _includeFile(self, content): - var = "_{}{}".format(self.varBase, self.count) - self.count += 1 - self.prolog.extend([ - "{VAR}=$(mktemp)".format(VAR=var), - "_BOB_TMP_CLEANUP+=( ${VAR} )".format(VAR=var), - "base64 -d > ${VAR} < {}".format(quote(BashLanguage.__munge(envFile))) if envFile else "", @@ -337,7 +330,6 @@ def __formatScript(spec): } declare -A _BOB_SOURCES=( [0]="Bob prolog" ) trap 'bob_handle_error $? >&2 ; exit 99' ERR - trap 'for i in "${_BOB_TMP_CLEANUP[@]-}" ; do /bin/rm -f "$i" ; done' EXIT set -o errtrace -o nounset -o pipefail """), BashLanguage.__formatSetup(spec), @@ -361,7 +353,7 @@ def __scriptFilePaths(spec, tmpDir): def setupShell(spec, tmpDir, keepEnv): realScriptFile, execScriptFile = BashLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(BashLanguage.__formatProlog(spec, keepEnv)) + f.write(BashLanguage.__formatProlog(spec, tmpDir, keepEnv)) f.write(BashLanguage.__formatSetup(spec)) args = ["bash", "--rcfile", BashLanguage.__munge(execScriptFile), "-s", "--"] @@ -372,7 +364,7 @@ def setupShell(spec, tmpDir, keepEnv): def setupCall(spec, tmpDir, keepEnv, trace): realScriptFile, execScriptFile = BashLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(BashLanguage.__formatScript(spec)) + f.write(BashLanguage.__formatScript(spec, tmpDir)) args = ["bash"] if trace: args.append("-x") @@ -410,17 +402,8 @@ def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): self.prolog = [] self.count = 0 - def _includeFile(self, content): - var = "$_{}{}".format(self.varBase, self.count) - self.count += 1 - self.prolog.append(dedent("""\ - {VAR} = (New-TemporaryFile).FullName - $_BOB_TMP_CLEANUP += {VAR} - [io.file]::WriteAllBytes({VAR}, [Convert]::FromBase64String(@'""" - .format(VAR=var))) - self.prolog.extend(sliceString(b64encode(content).decode("ascii"), 76)) - self.prolog.append("'@))") - return var + def _includeFile(self, name): + return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"' def _includeLiteral(self, content): return quotePwsh(content.decode('utf8')) @@ -448,7 +431,7 @@ class PwshLanguage: """) @staticmethod - def __formatProlog(spec): + def __formatProlog(spec, tmpDir): pathSep = ";" if sys.platform == "win32" else ":" env = { key: escapePwsh(value) for (key, value) in spec.env.items() } env.update({ @@ -474,6 +457,8 @@ def __formatProlog(spec): "$BOB_TOOL_PATHS=@{{ {} }}".format("; ".join(sorted( [ '{} = "{}"'.format(quotePwsh(name), escapePwsh(os.path.abspath(path))) for name,path in spec.toolPaths ] ))), + '$_BOB_TMP_BASE="{}"'.format(escapePwsh( + "/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp"))), "", "# Environment:", "\n".join('$Env:{}="{}"'.format(k, v) for (k,v) in sorted(env.items())), @@ -493,13 +478,13 @@ def __formatSetup(spec): ]) @staticmethod - def __formatScript(spec, trace): + def __formatScript(spec, tmpDir, trace): if spec.envFile: envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile) else: envFile = None ret = [ - PwshLanguage.__formatProlog(spec), + PwshLanguage.__formatProlog(spec, tmpDir), "", "# Setup", dedent("""\ @@ -523,20 +508,10 @@ def __formatScript(spec, trace): Set-PSDebug -Strict """), "", - dedent("""\ - try { - $_BOB_TMP_CLEANUP = @() - """), PwshLanguage.__formatSetup(spec), "", "# Recipe main script", spec.mainScript, - dedent("""\ - } finally { - foreach($f in $_BOB_TMP_CLEANUP) { - Remove-Item $f -Force - } - }"""), ] return "\n".join(ret) @@ -554,7 +529,7 @@ def __scriptFilePaths(spec, tmpDir): def setupShell(spec, tmpDir, keepEnv): realScriptFile, execScriptFile = PwshLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(PwshLanguage.__formatProlog(spec)) + f.write(PwshLanguage.__formatProlog(spec, tmpDir)) f.write(PwshLanguage.__formatSetup(spec)) interpreter = "powershell" if isWindows() else "pwsh" @@ -568,7 +543,7 @@ def setupShell(spec, tmpDir, keepEnv): def setupCall(spec, tmpDir, keepEnv, trace): realScriptFile, execScriptFile = PwshLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(PwshLanguage.__formatScript(spec, trace)) + f.write(PwshLanguage.__formatScript(spec, tmpDir, trace)) interpreter = "powershell" if isWindows() else "pwsh" args = [interpreter, "-ExecutionPolicy", "Bypass", "-File", execScriptFile] @@ -675,6 +650,11 @@ def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=F d['postRunCmds'] = step.getPostRunCmds() d['fingerprintScript'] = step._getFingerprintScript() + from base64 import b85encode + from zlib import compress + d['includedFiles'] = { name : b85encode(compress(data)).decode("ascii") + for name, data in step.getIncludedFiles().items() } + return self @classmethod @@ -800,3 +780,10 @@ def toolPaths(self): @property def scriptHint(self): return self.__data['scriptHint'] + + @property + def includedFiles(self): + from base64 import b85decode + from zlib import decompress + return { name : decompress(b85decode(data)) + for name, data in self.__data['includedFiles'].items() } From 1920662645faf8586f700a654e79aa140976485f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Tue, 15 Jun 2021 20:20:18 +0200 Subject: [PATCH 2/2] languages: add python support --- pym/bob/input.py | 34 +++++++--- pym/bob/languages.py | 151 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 168 insertions(+), 17 deletions(-) diff --git a/pym/bob/input.py b/pym/bob/input.py index ed296a1e8..922acf8f5 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -5,7 +5,7 @@ from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG from .errors import ParseError, BobError -from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage +from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage from .pathspec import PackageSet from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \ auditFromDir, getScm, SYNTHETIC_SCM_PROPS @@ -115,9 +115,11 @@ def fetchFingerprintScripts(recipe): recipe.get("fingerprintScript")), ScriptLanguage.PWSH : recipe.get("fingerprintScriptPwsh", recipe.get("fingerprintScript")), + ScriptLanguage.PYTHON : recipe.get("fingerprintScriptPython", + recipe.get("fingerprintScript")), } -def fetchScripts(recipe, prefix, resolveBash, resolvePwsh): +def fetchScripts(recipe, prefix, resolveBash, resolvePwsh, resolvePython): return { ScriptLanguage.BASH : ( resolveBash(recipe.get(prefix + "SetupBash", recipe.get(prefix + "Setup")), @@ -130,7 +132,13 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh): prefix + "Setup[Pwsh]"), resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")), prefix + "Script[Pwsh]"), - ) + ), + ScriptLanguage.PYTHON : ( + resolvePython(recipe.get(prefix + "SetupPython", recipe.get(prefix + "Setup")), + prefix + "Setup[Python]"), + resolvePython(recipe.get(prefix + "ScriptPython", recipe.get(prefix + "Script")), + prefix + "Script[Python]"), + ), } def mergeScripts(fragments, glue): @@ -2187,9 +2195,11 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b baseDir, packageName, sourceName).resolve incHelperPwsh = IncludeHelper(PwshLanguage, recipeSet.loadBinary, baseDir, packageName, sourceName).resolve + incHelperPython = IncludeHelper(PythonLanguage, recipeSet.loadBinary, + baseDir, packageName, sourceName).resolve self.__scriptLanguage = recipe.get("scriptLanguage") - self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh) + self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh, incHelperPython) self.__checkoutSCMs = recipe.get("checkoutSCM", []) for scm in self.__checkoutSCMs: scm["__source"] = sourceName @@ -2199,8 +2209,8 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b for a in self.__checkoutAsserts: a["__source"] = sourceName + ", checkoutAssert #{}".format(i) i += 1 - self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh) - self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh) + self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh, incHelperPython) + self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh, incHelperPython) self.__fingerprintScriptList = fetchFingerprintScripts(recipe) self.__fingerprintIf = recipe.get("fingerprintIf") self.__fingerprintVarsList = set(recipe.get("fingerprintVars", [])) @@ -3055,7 +3065,7 @@ class RecipeSet: ), schema.Optional('layers') : [str], schema.Optional('scriptLanguage', - default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"), + default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell", "python"), schema.Use(ScriptLanguage)), }) @@ -3653,21 +3663,27 @@ def __createSchemas(self): schema.Optional('checkoutScript') : str, schema.Optional('checkoutScriptBash') : str, schema.Optional('checkoutScriptPwsh') : str, + schema.Optional('checkoutScriptPython') : str, schema.Optional('checkoutSetup') : str, schema.Optional('checkoutSetupBash') : str, schema.Optional('checkoutSetupPwsh') : str, + schema.Optional('checkoutSetupPython') : str, schema.Optional('buildScript') : str, schema.Optional('buildScriptBash') : str, schema.Optional('buildScriptPwsh') : str, + schema.Optional('buildScriptPython') : str, schema.Optional('buildSetup') : str, schema.Optional('buildSetupBash') : str, schema.Optional('buildSetupPwsh') : str, + schema.Optional('buildSetupPython') : str, schema.Optional('packageScript') : str, schema.Optional('packageScriptBash') : str, schema.Optional('packageScriptPwsh') : str, + schema.Optional('packageScriptPython') : str, schema.Optional('packageSetup') : str, schema.Optional('packageSetupBash') : str, schema.Optional('packageSetupPwsh') : str, + schema.Optional('packageSetupPython') : str, schema.Optional('checkoutTools') : [ toolNameSchema ], schema.Optional('buildTools') : [ toolNameSchema ], schema.Optional('packageTools') : [ toolNameSchema ], @@ -3705,6 +3721,7 @@ def __createSchemas(self): schema.Optional('fingerprintScript', default="") : str, schema.Optional('fingerprintScriptBash') : str, schema.Optional('fingerprintScriptPwsh', default="") : str, + schema.Optional('fingerprintScriptPython', default="") : str, schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression), schema.Optional('fingerprintVars') : [ varNameUseSchema ], }) @@ -3725,9 +3742,10 @@ def __createSchemas(self): schema.Optional('fingerprintScript', default="") : str, schema.Optional('fingerprintScriptBash') : str, schema.Optional('fingerprintScriptPwsh', default="") : str, + schema.Optional('fingerprintScriptPython', default="") : str, schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression), schema.Optional('fingerprintVars') : [ varNameUseSchema ], - schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell"), + schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell", "python"), schema.Use(ScriptLanguage)), schema.Optional('jobServer') : bool, } diff --git a/pym/bob/languages.py b/pym/bob/languages.py index 58dadbdf2..17edb50f3 100644 --- a/pym/bob/languages.py +++ b/pym/bob/languages.py @@ -139,6 +139,7 @@ class ScriptLanguage(Enum): BASH = 'bash' PWSH = 'PowerShell' + PYTHON = 'python' class IncludeResolver: @@ -185,7 +186,7 @@ def _includeLiteral(self, content): raise NotImplementedError() def _resolveContent(self, result): - raise NotImplementedError() + return result def resolve(self, result): return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles) @@ -397,20 +398,12 @@ def setupFingerprint(spec, env): class PwshResolver(IncludeResolver): - def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): - super().__init__(fileLoader, baseDir, origText, sourceName, varBase) - self.prolog = [] - self.count = 0 - def _includeFile(self, name): return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"' def _includeLiteral(self, content): return quotePwsh(content.decode('utf8')) - def _resolveContent(self, result): - return "\n".join(self.prolog + [result]) - class PwshLanguage: index = ScriptLanguage.PWSH @@ -574,9 +567,149 @@ def setupFingerprint(spec, env): return [interpreter, "-c", spec.fingerprintScript] +class PythonResolver(IncludeResolver): + def _includeFile(self, name): + return 'os.path.join(_BOB_TMP_BASE, ' + repr(name) + ')' + + def _includeLiteral(self, content): + return repr(content.decode('utf8')) + + +class PythonLanguage: + index = ScriptLanguage.PYTHON + glue = "\nos.chdir(os.environ['BOB_CWD'])\n" + Resolver = PythonResolver + + PROLOGUE = dedent("""\ + import os, os.path, sys + """) + + @staticmethod + def __formatProlog(spec, tmpDir): + pathSep = ";" if sys.platform == "win32" else ":" + env = { key : repr(value) for (key, value) in spec.env.items() } + env.update({ + "PATH": '"' + pathSep + '".join([' + ", ".join( + [repr(os.path.abspath(p)) for p in spec.paths] + + (['os.environ["PATH"]'] if not spec.hasSandbox else + [repr(p) for p in spec.sandboxPaths]) + ) + '])', + "LD_LIBRARY_PATH": '"' + pathSep + '".join(' + + repr([ os.path.abspath(p) for p in spec.libraryPaths ]) + ')', + "BOB_CWD": repr(os.path.abspath(spec.workspaceExecPath)), + }) + + ret = [ + "# Convenience helpers", + PythonLanguage.PROLOGUE, + "", + "# Special Bob array variables:", + "BOB_ALL_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name, path in spec.allPaths ] + ))), + "BOB_DEP_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name,path in spec.depPaths ] + ))), + "BOB_TOOL_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name,path in spec.toolPaths ] + ))), + '_BOB_TMP_BASE = ' + repr("/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp")), + "", + "# Environment:", + "\n".join('os.environ["{}"] = {}'.format(k, v) for (k,v) in sorted(env.items())), + ] + return "\n".join(ret) + + @staticmethod + def __formatSetup(spec): + return "\n".join([ + "", + "# Recipe setup script", + spec.setupScript, + "os.chdir(os.environ['BOB_CWD'])", + ]) + + @staticmethod + def __formatScript(spec, tmpDir, trace): + if spec.envFile: + envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile) + else: + envFile = None + ret = [ + PythonLanguage.__formatProlog(spec, tmpDir), + "", + "# Setup", + dedent("""\ + with open({ENV_FILE}, "w") as f: + f.write(repr({{ "env" : dict(os.environ), "globals" : globals() }})) + """.format(ENV_FILE=repr(envFile))) if envFile else "", + "os.chdir(os.environ['BOB_CWD'])", + "", + PythonLanguage.__formatSetup(spec), + "", + "# Recipe main script", + spec.mainScript, + ] + return "\n".join(ret) + + @staticmethod + def __scriptFilePaths(spec, tmpDir): + if spec.hasSandbox: + execScriptFile = "/.script.py" + realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".py" + else: + execScriptFile = (spec.scriptHint or os.path.join(tmpDir, "script")) + ".py" + realScriptFile = execScriptFile + return (os.path.abspath(realScriptFile), os.path.abspath(execScriptFile)) + + @staticmethod + def setupShell(spec, tmpDir, keepEnv): + realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir) + with open(realScriptFile, "w") as f: + f.write(PythonLanguage.__formatProlog(spec, tmpDir)) + f.write(PythonLanguage.__formatSetup(spec)) + + args = [sys.executable, "-i", execScriptFile] + args.extend(os.path.abspath(a) for a in spec.args) + + return (realScriptFile, execScriptFile, args) + + @staticmethod + def setupCall(spec, tmpDir, keepEnv, trace): + realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir) + with open(realScriptFile, "w") as f: + f.write(PythonLanguage.__formatScript(spec, tmpDir, trace)) + + args = [sys.executable, execScriptFile] + args.extend(os.path.abspath(a) for a in spec.args) + + return (realScriptFile, execScriptFile, args) + + @staticmethod + def mangleFingerprints(scriptFragments, env): + # join the script fragments first + script = joinScripts(scriptFragments, PythonLanguage.glue) + + # do not add preamble for empty scripts + if not script: return "" + + # Add snippets as they match and a default settings preamble + ret = [script] + for n,v in sorted(env.items()): + ret.append('os.environ["{}"] = {}'.format(k, repr(v))) + ret.append(PythonLanguage.PROLOGUE) + + return "\n".join(reversed(ret)) + + @staticmethod + def setupFingerprint(spec, env): + return [sys.executable, "-c", spec.fingerprintScript] + + LANG = { ScriptLanguage.BASH : BashLanguage, ScriptLanguage.PWSH : PwshLanguage, + ScriptLanguage.PYTHON : PythonLanguage, } def getLanguage(language):