diff --git a/pym/bob/input.py b/pym/bob/input.py index 9281579f3..922acf8f5 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -5,7 +5,7 @@ from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG from .errors import ParseError, BobError -from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage +from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage from .pathspec import PackageSet from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \ auditFromDir, getScm, SYNTHETIC_SCM_PROPS @@ -115,9 +115,11 @@ def fetchFingerprintScripts(recipe): recipe.get("fingerprintScript")), ScriptLanguage.PWSH : recipe.get("fingerprintScriptPwsh", recipe.get("fingerprintScript")), + ScriptLanguage.PYTHON : recipe.get("fingerprintScriptPython", + recipe.get("fingerprintScript")), } -def fetchScripts(recipe, prefix, resolveBash, resolvePwsh): +def fetchScripts(recipe, prefix, resolveBash, resolvePwsh, resolvePython): return { ScriptLanguage.BASH : ( resolveBash(recipe.get(prefix + "SetupBash", recipe.get(prefix + "Setup")), @@ -130,21 +132,31 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh): prefix + "Setup[Pwsh]"), resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")), prefix + "Script[Pwsh]"), - ) + ), + ScriptLanguage.PYTHON : ( + resolvePython(recipe.get(prefix + "SetupPython", recipe.get(prefix + "Setup")), + prefix + "Setup[Python]"), + resolvePython(recipe.get(prefix + "ScriptPython", recipe.get(prefix + "Script")), + prefix + "Script[Python]"), + ), } def mergeScripts(fragments, glue): """Join all scripts of the recipe and its classes. - The result is a tuple with (setupScript, mainScript, digestScript) + The result is a tuple with (setupScript, mainScript, digestScript, includedFiles) """ + return ( joinScripts((f[0][0] for f in fragments), glue), joinScripts((f[1][0] for f in fragments), glue), joinScripts( ( joinScripts((f[0][1] for f in fragments), "\n"), joinScripts((f[1][1] for f in fragments), "\n"), - ), "\n") + ), "\n"), + { name : content for name, content in + chain.from_iterable(chain(f[0][2].items(), f[1][2].items()) for f in fragments) + }, ) @@ -790,6 +802,9 @@ def getPostRunCmds(self): def getDigestScript(self): raise NotImplementedError + def getIncludedFiles(self): + raise NotImplementedError + def getLabel(self): raise NotImplementedError @@ -1042,6 +1057,9 @@ def getDigestScript(self): """ return self._coreStep.getDigestScript() + def getIncludedFiles(self): + return self._coreStep.getIncludedFiles() + def isDeterministic(self): """Return whether the step is deterministic. @@ -1458,6 +1476,9 @@ def getDigestScript(self): else: return None + def getIncludedFiles(self): + return self.corePackage.recipe.checkoutIncludedFiles + @property def fingerprintMask(self): return 0 @@ -1537,7 +1558,7 @@ def hasNetAccess(self): class CoreBuildStep(CoreStep): __slots__ = [] - def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]): + def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]): isValid = script[1] is not None super().__init__(corePackage, isValid, True, digestEnv, env, args) @@ -1568,6 +1589,9 @@ def getMainScript(self): def getDigestScript(self): return self.corePackage.recipe.buildDigestScript + def getIncludedFiles(self): + return self.corePackage.recipe.buildIncludedFiles + @property def fingerprintMask(self): # Remove bits of all tools that are not used in buildStep @@ -1591,7 +1615,7 @@ def hasNetAccess(self): class CorePackageStep(CoreStep): __slots__ = [] - def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]): + def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]): isValid = script[1] is not None super().__init__(corePackage, isValid, True, digestEnv, env, args) @@ -1622,6 +1646,9 @@ def getMainScript(self): def getDigestScript(self): return self.corePackage.recipe.packageDigestScript + def getIncludedFiles(self): + return self.corePackage.recipe.packageIncludedFiles + @property def fingerprintMask(self): return self.corePackage.fingerprintMask @@ -1884,7 +1911,7 @@ def resolve(self, text, section): raise ParseError("Bad substiturion in {}: {}".format(section, str(e))) return resolver.resolve(ret) else: - return (None, None) + return (None, None, {}) def mergeFilter(left, right): if left is None: @@ -2168,9 +2195,11 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b baseDir, packageName, sourceName).resolve incHelperPwsh = IncludeHelper(PwshLanguage, recipeSet.loadBinary, baseDir, packageName, sourceName).resolve + incHelperPython = IncludeHelper(PythonLanguage, recipeSet.loadBinary, + baseDir, packageName, sourceName).resolve self.__scriptLanguage = recipe.get("scriptLanguage") - self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh) + self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh, incHelperPython) self.__checkoutSCMs = recipe.get("checkoutSCM", []) for scm in self.__checkoutSCMs: scm["__source"] = sourceName @@ -2180,8 +2209,8 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b for a in self.__checkoutAsserts: a["__source"] = sourceName + ", checkoutAssert #{}".format(i) i += 1 - self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh) - self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh) + self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh, incHelperPython) + self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh, incHelperPython) self.__fingerprintScriptList = fetchFingerprintScripts(recipe) self.__fingerprintIf = recipe.get("fingerprintIf") self.__fingerprintVarsList = set(recipe.get("fingerprintVars", [])) @@ -2316,7 +2345,7 @@ def coDet(r): # the package step must always be valid if self.__package[1] is None: - self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709') + self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709', {}) # final shared value self.__shared = self.__shared == True @@ -2643,7 +2672,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, directPackages, indirectPackages, states, uidGen(), doFingerprint) # optional checkout step - if self.__checkout != (None, None, None) or self.__checkoutSCMs or self.__checkoutAsserts: + if self.__checkout != (None, None, None, {}) or self.__checkoutSCMs or self.__checkoutAsserts: checkoutDigestEnv = env.prune(self.__checkoutVars) checkoutEnv = ( env.prune(self.__checkoutVars | self.__checkoutVarsWeak) if self.__checkoutVarsWeak else checkoutDigestEnv ) @@ -2653,7 +2682,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, srcCoreStep = p.createInvalidCoreCheckoutStep() # optional build step - if self.__build != (None, None, None): + if self.__build != (None, None, None, {}): buildDigestEnv = env.prune(self.__buildVars) buildEnv = ( env.prune(self.__buildVars | self.__buildVarsWeak) if self.__buildVarsWeak else buildDigestEnv ) @@ -2765,6 +2794,10 @@ def checkoutMainScript(self): def checkoutDigestScript(self): return self.__checkout[2] or "" + @property + def checkoutIncludedFiles(self): + return self.__checkout[3] + @property def checkoutDeterministic(self): return self.__checkoutDeterministic @@ -2793,6 +2826,10 @@ def buildMainScript(self): def buildDigestScript(self): return self.__build[2] + @property + def buildIncludedFiles(self): + return self.__build[3] + @property def buildVars(self): return self.__buildVars @@ -2813,6 +2850,10 @@ def packageMainScript(self): def packageDigestScript(self): return self.__package[2] + @property + def packageIncludedFiles(self): + return self.__package[3] + @property def packageVars(self): return self.__packageVars @@ -3024,7 +3065,7 @@ class RecipeSet: ), schema.Optional('layers') : [str], schema.Optional('scriptLanguage', - default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"), + default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell", "python"), schema.Use(ScriptLanguage)), }) @@ -3622,21 +3663,27 @@ def __createSchemas(self): schema.Optional('checkoutScript') : str, schema.Optional('checkoutScriptBash') : str, schema.Optional('checkoutScriptPwsh') : str, + schema.Optional('checkoutScriptPython') : str, schema.Optional('checkoutSetup') : str, schema.Optional('checkoutSetupBash') : str, schema.Optional('checkoutSetupPwsh') : str, + schema.Optional('checkoutSetupPython') : str, schema.Optional('buildScript') : str, schema.Optional('buildScriptBash') : str, schema.Optional('buildScriptPwsh') : str, + schema.Optional('buildScriptPython') : str, schema.Optional('buildSetup') : str, schema.Optional('buildSetupBash') : str, schema.Optional('buildSetupPwsh') : str, + schema.Optional('buildSetupPython') : str, schema.Optional('packageScript') : str, schema.Optional('packageScriptBash') : str, schema.Optional('packageScriptPwsh') : str, + schema.Optional('packageScriptPython') : str, schema.Optional('packageSetup') : str, schema.Optional('packageSetupBash') : str, schema.Optional('packageSetupPwsh') : str, + schema.Optional('packageSetupPython') : str, schema.Optional('checkoutTools') : [ toolNameSchema ], schema.Optional('buildTools') : [ toolNameSchema ], schema.Optional('packageTools') : [ toolNameSchema ], @@ -3674,6 +3721,7 @@ def __createSchemas(self): schema.Optional('fingerprintScript', default="") : str, schema.Optional('fingerprintScriptBash') : str, schema.Optional('fingerprintScriptPwsh', default="") : str, + schema.Optional('fingerprintScriptPython', default="") : str, schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression), schema.Optional('fingerprintVars') : [ varNameUseSchema ], }) @@ -3694,9 +3742,10 @@ def __createSchemas(self): schema.Optional('fingerprintScript', default="") : str, schema.Optional('fingerprintScriptBash') : str, schema.Optional('fingerprintScriptPwsh', default="") : str, + schema.Optional('fingerprintScriptPython', default="") : str, schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression), schema.Optional('fingerprintVars') : [ varNameUseSchema ], - schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell"), + schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell", "python"), schema.Use(ScriptLanguage)), schema.Optional('jobServer') : bool, } diff --git a/pym/bob/invoker.py b/pym/bob/invoker.py index 3e56bfa9e..24888bb12 100644 --- a/pym/bob/invoker.py +++ b/pym/bob/invoker.py @@ -19,6 +19,7 @@ import os import re import shutil +import stat import subprocess import sys import tempfile @@ -266,7 +267,6 @@ def __getSandboxCmds(self, tmpDir): #FIXME: if verbosity >= 4: cmdArgs.append('-D') cmdArgs.extend(["-S", tmpDir]) cmdArgs.extend(["-H", "bob"]) - cmdArgs.extend(["-d", "/tmp"]) sandboxRootFs = os.path.abspath(self.__spec.sandboxRootWorkspace) for f in os.listdir(sandboxRootFs): cmdArgs.extend(["-M", os.path.join(sandboxRootFs, f), "-m", "/"+f]) @@ -301,6 +301,16 @@ async def executeStep(self, mode, clean=False, keepSandbox=False): # also used as ephemeral sandbox container. tmpDir = tempfile.mkdtemp() + # Create /tmp in sandbox container. Receives included files and + # needs to be there anyway. + os.mkdir(os.path.join(tmpDir, "tmp"), 0o755) + for name, content in self.__spec.includedFiles.items(): + fn = os.path.join(tmpDir, "tmp", name) + with open(fn, "wb") as f: + f.write(content) + # Set explicit mode to retain Bob 0.19 behaviour. + os.chmod(fn, stat.S_IREAD|stat.S_IWRITE) + # prepare workspace clean = self.__spec.clean if self.__spec.clean is not None else clean if not os.path.isdir(self.__spec.workspaceWorkspacePath): @@ -468,6 +478,7 @@ async def executeFingerprint(self, keepSandbox=False): # Setup workspace cmdArgs = self.__getSandboxCmds(tmpDir) + cmdArgs.extend(["-d", "/tmp"]) cmdArgs.extend(["-d", "/bob/fingerprint"]) cmdArgs.extend(["-W", "/bob/fingerprint"]) diff --git a/pym/bob/languages.py b/pym/bob/languages.py index efc89d9a2..17edb50f3 100644 --- a/pym/bob/languages.py +++ b/pym/bob/languages.py @@ -6,8 +6,7 @@ from . import BOB_INPUT_HASH from .errors import ParseError from .utils import escapePwsh, quotePwsh, isWindows, asHexStr -from .utils import joinScripts, sliceString -from base64 import b64encode +from .utils import joinScripts, sliceString, INVALID_CHAR_TRANS from enum import Enum from glob import glob from shlex import quote @@ -140,6 +139,7 @@ class ScriptLanguage(Enum): BASH = 'bash' PWSH = 'PowerShell' + PYTHON = 'python' class IncludeResolver: @@ -148,7 +148,9 @@ def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): self.baseDir = baseDir self.sourceName = sourceName self.varBase = varBase + self.count = 0 self.__incDigests = [ hashlib.sha1(origText.encode('utf8')).digest().hex() ] + self.__incFiles = {} def __getitem__(self, item): mode = item[0] @@ -167,24 +169,27 @@ def __getitem__(self, item): self.__incDigests.append(hashlib.sha1(content).digest().hex()) if mode == '<': - ret = self._includeFile(content) + name = "{}_{}".format(self.varBase, self.count).translate(INVALID_CHAR_TRANS) + self.count += 1 + self.__incFiles[name] = content + ret = self._includeFile(name) else: assert mode == "'" ret = self._includeLiteral(content) return ret - def _includeFile(self, content): + def _includeFile(self, name): raise NotImplementedError() def _includeLiteral(self, content): raise NotImplementedError() def _resolveContent(self, result): - raise NotImplementedError() + return result def resolve(self, result): - return (self._resolveContent(result), "\n".join(self.__incDigests)) + return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles) # For each supported language the following runtime environments must be # considered: @@ -197,28 +202,15 @@ def resolve(self, result): # converted. class BashResolver(IncludeResolver): - def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): - super().__init__(fileLoader, baseDir, origText, sourceName, varBase) - self.prolog = [] - self.count = 0 - def _includeFile(self, content): - var = "_{}{}".format(self.varBase, self.count) - self.count += 1 - self.prolog.extend([ - "{VAR}=$(mktemp)".format(VAR=var), - "_BOB_TMP_CLEANUP+=( ${VAR} )".format(VAR=var), - "base64 -d > ${VAR} < {}".format(quote(BashLanguage.__munge(envFile))) if envFile else "", @@ -337,7 +331,6 @@ def __formatScript(spec): } declare -A _BOB_SOURCES=( [0]="Bob prolog" ) trap 'bob_handle_error $? >&2 ; exit 99' ERR - trap 'for i in "${_BOB_TMP_CLEANUP[@]-}" ; do /bin/rm -f "$i" ; done' EXIT set -o errtrace -o nounset -o pipefail """), BashLanguage.__formatSetup(spec), @@ -361,7 +354,7 @@ def __scriptFilePaths(spec, tmpDir): def setupShell(spec, tmpDir, keepEnv): realScriptFile, execScriptFile = BashLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(BashLanguage.__formatProlog(spec, keepEnv)) + f.write(BashLanguage.__formatProlog(spec, tmpDir, keepEnv)) f.write(BashLanguage.__formatSetup(spec)) args = ["bash", "--rcfile", BashLanguage.__munge(execScriptFile), "-s", "--"] @@ -372,7 +365,7 @@ def setupShell(spec, tmpDir, keepEnv): def setupCall(spec, tmpDir, keepEnv, trace): realScriptFile, execScriptFile = BashLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(BashLanguage.__formatScript(spec)) + f.write(BashLanguage.__formatScript(spec, tmpDir)) args = ["bash"] if trace: args.append("-x") @@ -405,29 +398,12 @@ def setupFingerprint(spec, env): class PwshResolver(IncludeResolver): - def __init__(self, fileLoader, baseDir, origText, sourceName, varBase): - super().__init__(fileLoader, baseDir, origText, sourceName, varBase) - self.prolog = [] - self.count = 0 - - def _includeFile(self, content): - var = "$_{}{}".format(self.varBase, self.count) - self.count += 1 - self.prolog.append(dedent("""\ - {VAR} = (New-TemporaryFile).FullName - $_BOB_TMP_CLEANUP += {VAR} - [io.file]::WriteAllBytes({VAR}, [Convert]::FromBase64String(@'""" - .format(VAR=var))) - self.prolog.extend(sliceString(b64encode(content).decode("ascii"), 76)) - self.prolog.append("'@))") - return var + def _includeFile(self, name): + return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"' def _includeLiteral(self, content): return quotePwsh(content.decode('utf8')) - def _resolveContent(self, result): - return "\n".join(self.prolog + [result]) - class PwshLanguage: index = ScriptLanguage.PWSH @@ -448,7 +424,7 @@ class PwshLanguage: """) @staticmethod - def __formatProlog(spec): + def __formatProlog(spec, tmpDir): pathSep = ";" if sys.platform == "win32" else ":" env = { key: escapePwsh(value) for (key, value) in spec.env.items() } env.update({ @@ -474,6 +450,8 @@ def __formatProlog(spec): "$BOB_TOOL_PATHS=@{{ {} }}".format("; ".join(sorted( [ '{} = "{}"'.format(quotePwsh(name), escapePwsh(os.path.abspath(path))) for name,path in spec.toolPaths ] ))), + '$_BOB_TMP_BASE="{}"'.format(escapePwsh( + "/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp"))), "", "# Environment:", "\n".join('$Env:{}="{}"'.format(k, v) for (k,v) in sorted(env.items())), @@ -493,13 +471,13 @@ def __formatSetup(spec): ]) @staticmethod - def __formatScript(spec, trace): + def __formatScript(spec, tmpDir, trace): if spec.envFile: envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile) else: envFile = None ret = [ - PwshLanguage.__formatProlog(spec), + PwshLanguage.__formatProlog(spec, tmpDir), "", "# Setup", dedent("""\ @@ -523,20 +501,10 @@ def __formatScript(spec, trace): Set-PSDebug -Strict """), "", - dedent("""\ - try { - $_BOB_TMP_CLEANUP = @() - """), PwshLanguage.__formatSetup(spec), "", "# Recipe main script", spec.mainScript, - dedent("""\ - } finally { - foreach($f in $_BOB_TMP_CLEANUP) { - Remove-Item $f -Force - } - }"""), ] return "\n".join(ret) @@ -554,7 +522,7 @@ def __scriptFilePaths(spec, tmpDir): def setupShell(spec, tmpDir, keepEnv): realScriptFile, execScriptFile = PwshLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(PwshLanguage.__formatProlog(spec)) + f.write(PwshLanguage.__formatProlog(spec, tmpDir)) f.write(PwshLanguage.__formatSetup(spec)) interpreter = "powershell" if isWindows() else "pwsh" @@ -568,7 +536,7 @@ def setupShell(spec, tmpDir, keepEnv): def setupCall(spec, tmpDir, keepEnv, trace): realScriptFile, execScriptFile = PwshLanguage.__scriptFilePaths(spec, tmpDir) with open(realScriptFile, "w") as f: - f.write(PwshLanguage.__formatScript(spec, trace)) + f.write(PwshLanguage.__formatScript(spec, tmpDir, trace)) interpreter = "powershell" if isWindows() else "pwsh" args = [interpreter, "-ExecutionPolicy", "Bypass", "-File", execScriptFile] @@ -599,9 +567,149 @@ def setupFingerprint(spec, env): return [interpreter, "-c", spec.fingerprintScript] +class PythonResolver(IncludeResolver): + def _includeFile(self, name): + return 'os.path.join(_BOB_TMP_BASE, ' + repr(name) + ')' + + def _includeLiteral(self, content): + return repr(content.decode('utf8')) + + +class PythonLanguage: + index = ScriptLanguage.PYTHON + glue = "\nos.chdir(os.environ['BOB_CWD'])\n" + Resolver = PythonResolver + + PROLOGUE = dedent("""\ + import os, os.path, sys + """) + + @staticmethod + def __formatProlog(spec, tmpDir): + pathSep = ";" if sys.platform == "win32" else ":" + env = { key : repr(value) for (key, value) in spec.env.items() } + env.update({ + "PATH": '"' + pathSep + '".join([' + ", ".join( + [repr(os.path.abspath(p)) for p in spec.paths] + + (['os.environ["PATH"]'] if not spec.hasSandbox else + [repr(p) for p in spec.sandboxPaths]) + ) + '])', + "LD_LIBRARY_PATH": '"' + pathSep + '".join(' + + repr([ os.path.abspath(p) for p in spec.libraryPaths ]) + ')', + "BOB_CWD": repr(os.path.abspath(spec.workspaceExecPath)), + }) + + ret = [ + "# Convenience helpers", + PythonLanguage.PROLOGUE, + "", + "# Special Bob array variables:", + "BOB_ALL_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name, path in spec.allPaths ] + ))), + "BOB_DEP_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name,path in spec.depPaths ] + ))), + "BOB_TOOL_PATHS = dict({})".format(repr(sorted( + [ (name, os.path.abspath(path)) for name,path in spec.toolPaths ] + ))), + '_BOB_TMP_BASE = ' + repr("/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp")), + "", + "# Environment:", + "\n".join('os.environ["{}"] = {}'.format(k, v) for (k,v) in sorted(env.items())), + ] + return "\n".join(ret) + + @staticmethod + def __formatSetup(spec): + return "\n".join([ + "", + "# Recipe setup script", + spec.setupScript, + "os.chdir(os.environ['BOB_CWD'])", + ]) + + @staticmethod + def __formatScript(spec, tmpDir, trace): + if spec.envFile: + envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile) + else: + envFile = None + ret = [ + PythonLanguage.__formatProlog(spec, tmpDir), + "", + "# Setup", + dedent("""\ + with open({ENV_FILE}, "w") as f: + f.write(repr({{ "env" : dict(os.environ), "globals" : globals() }})) + """.format(ENV_FILE=repr(envFile))) if envFile else "", + "os.chdir(os.environ['BOB_CWD'])", + "", + PythonLanguage.__formatSetup(spec), + "", + "# Recipe main script", + spec.mainScript, + ] + return "\n".join(ret) + + @staticmethod + def __scriptFilePaths(spec, tmpDir): + if spec.hasSandbox: + execScriptFile = "/.script.py" + realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".py" + else: + execScriptFile = (spec.scriptHint or os.path.join(tmpDir, "script")) + ".py" + realScriptFile = execScriptFile + return (os.path.abspath(realScriptFile), os.path.abspath(execScriptFile)) + + @staticmethod + def setupShell(spec, tmpDir, keepEnv): + realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir) + with open(realScriptFile, "w") as f: + f.write(PythonLanguage.__formatProlog(spec, tmpDir)) + f.write(PythonLanguage.__formatSetup(spec)) + + args = [sys.executable, "-i", execScriptFile] + args.extend(os.path.abspath(a) for a in spec.args) + + return (realScriptFile, execScriptFile, args) + + @staticmethod + def setupCall(spec, tmpDir, keepEnv, trace): + realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir) + with open(realScriptFile, "w") as f: + f.write(PythonLanguage.__formatScript(spec, tmpDir, trace)) + + args = [sys.executable, execScriptFile] + args.extend(os.path.abspath(a) for a in spec.args) + + return (realScriptFile, execScriptFile, args) + + @staticmethod + def mangleFingerprints(scriptFragments, env): + # join the script fragments first + script = joinScripts(scriptFragments, PythonLanguage.glue) + + # do not add preamble for empty scripts + if not script: return "" + + # Add snippets as they match and a default settings preamble + ret = [script] + for n,v in sorted(env.items()): + ret.append('os.environ["{}"] = {}'.format(k, repr(v))) + ret.append(PythonLanguage.PROLOGUE) + + return "\n".join(reversed(ret)) + + @staticmethod + def setupFingerprint(spec, env): + return [sys.executable, "-c", spec.fingerprintScript] + + LANG = { ScriptLanguage.BASH : BashLanguage, ScriptLanguage.PWSH : PwshLanguage, + ScriptLanguage.PYTHON : PythonLanguage, } def getLanguage(language): @@ -675,6 +783,11 @@ def fromStep(cls, step, envFile=None, envWhiteList=[], logFile=None, isJenkins=F d['postRunCmds'] = step.getPostRunCmds() d['fingerprintScript'] = step._getFingerprintScript() + from base64 import b85encode + from zlib import compress + d['includedFiles'] = { name : b85encode(compress(data)).decode("ascii") + for name, data in step.getIncludedFiles().items() } + return self @classmethod @@ -800,3 +913,10 @@ def toolPaths(self): @property def scriptHint(self): return self.__data['scriptHint'] + + @property + def includedFiles(self): + from base64 import b85decode + from zlib import decompress + return { name : decompress(b85decode(data)) + for name, data in self.__data['includedFiles'].items() }