Skip to content

Commit

Permalink
languages: add python support
Browse files Browse the repository at this point in the history
  • Loading branch information
@jkloetzke
jkloetzke committed Jun 22, 2021
1 parent 2b13716 commit 1920662
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 17 deletions.
34 changes: 26 additions & 8 deletions pym/bob/input.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG
from .errors import ParseError, BobError
from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage
from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage
from .pathspec import PackageSet
from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
auditFromDir, getScm, SYNTHETIC_SCM_PROPS
Expand Down Expand Up @@ -115,9 +115,11 @@ def fetchFingerprintScripts(recipe):
recipe.get("fingerprintScript")),
ScriptLanguage.PWSH : recipe.get("fingerprintScriptPwsh",
recipe.get("fingerprintScript")),
ScriptLanguage.PYTHON : recipe.get("fingerprintScriptPython",
recipe.get("fingerprintScript")),
}

def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
def fetchScripts(recipe, prefix, resolveBash, resolvePwsh, resolvePython):
return {
ScriptLanguage.BASH : (
resolveBash(recipe.get(prefix + "SetupBash", recipe.get(prefix + "Setup")),
Expand All @@ -130,7 +132,13 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
prefix + "Setup[Pwsh]"),
resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")),
prefix + "Script[Pwsh]"),
)
),
ScriptLanguage.PYTHON : (
resolvePython(recipe.get(prefix + "SetupPython", recipe.get(prefix + "Setup")),
prefix + "Setup[Python]"),
resolvePython(recipe.get(prefix + "ScriptPython", recipe.get(prefix + "Script")),
prefix + "Script[Python]"),
),
}

def mergeScripts(fragments, glue):
Expand Down Expand Up @@ -2187,9 +2195,11 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
baseDir, packageName, sourceName).resolve
incHelperPwsh = IncludeHelper(PwshLanguage, recipeSet.loadBinary,
baseDir, packageName, sourceName).resolve
incHelperPython = IncludeHelper(PythonLanguage, recipeSet.loadBinary,
baseDir, packageName, sourceName).resolve

self.__scriptLanguage = recipe.get("scriptLanguage")
self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh)
self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh, incHelperPython)
self.__checkoutSCMs = recipe.get("checkoutSCM", [])
for scm in self.__checkoutSCMs:
scm["__source"] = sourceName
Expand All @@ -2199,8 +2209,8 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
for a in self.__checkoutAsserts:
a["__source"] = sourceName + ", checkoutAssert #{}".format(i)
i += 1
self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh)
self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh)
self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh, incHelperPython)
self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh, incHelperPython)
self.__fingerprintScriptList = fetchFingerprintScripts(recipe)
self.__fingerprintIf = recipe.get("fingerprintIf")
self.__fingerprintVarsList = set(recipe.get("fingerprintVars", []))
Expand Down Expand Up @@ -3055,7 +3065,7 @@ class RecipeSet:
),
schema.Optional('layers') : [str],
schema.Optional('scriptLanguage',
default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell", "python"),
schema.Use(ScriptLanguage)),
})

Expand Down Expand Up @@ -3653,21 +3663,27 @@ def __createSchemas(self):
schema.Optional('checkoutScript') : str,
schema.Optional('checkoutScriptBash') : str,
schema.Optional('checkoutScriptPwsh') : str,
schema.Optional('checkoutScriptPython') : str,
schema.Optional('checkoutSetup') : str,
schema.Optional('checkoutSetupBash') : str,
schema.Optional('checkoutSetupPwsh') : str,
schema.Optional('checkoutSetupPython') : str,
schema.Optional('buildScript') : str,
schema.Optional('buildScriptBash') : str,
schema.Optional('buildScriptPwsh') : str,
schema.Optional('buildScriptPython') : str,
schema.Optional('buildSetup') : str,
schema.Optional('buildSetupBash') : str,
schema.Optional('buildSetupPwsh') : str,
schema.Optional('buildSetupPython') : str,
schema.Optional('packageScript') : str,
schema.Optional('packageScriptBash') : str,
schema.Optional('packageScriptPwsh') : str,
schema.Optional('packageScriptPython') : str,
schema.Optional('packageSetup') : str,
schema.Optional('packageSetupBash') : str,
schema.Optional('packageSetupPwsh') : str,
schema.Optional('packageSetupPython') : str,
schema.Optional('checkoutTools') : [ toolNameSchema ],
schema.Optional('buildTools') : [ toolNameSchema ],
schema.Optional('packageTools') : [ toolNameSchema ],
Expand Down Expand Up @@ -3705,6 +3721,7 @@ def __createSchemas(self):
schema.Optional('fingerprintScript', default="") : str,
schema.Optional('fingerprintScriptBash') : str,
schema.Optional('fingerprintScriptPwsh', default="") : str,
schema.Optional('fingerprintScriptPython', default="") : str,
schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression),
schema.Optional('fingerprintVars') : [ varNameUseSchema ],
})
Expand All @@ -3725,9 +3742,10 @@ def __createSchemas(self):
schema.Optional('fingerprintScript', default="") : str,
schema.Optional('fingerprintScriptBash') : str,
schema.Optional('fingerprintScriptPwsh', default="") : str,
schema.Optional('fingerprintScriptPython', default="") : str,
schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression),
schema.Optional('fingerprintVars') : [ varNameUseSchema ],
schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell"),
schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell", "python"),
schema.Use(ScriptLanguage)),
schema.Optional('jobServer') : bool,
}
Expand Down
151 changes: 142 additions & 9 deletions pym/bob/languages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@
class ScriptLanguage(Enum):
BASH = 'bash'
PWSH = 'PowerShell'
PYTHON = 'python'


class IncludeResolver:
Expand Down Expand Up @@ -185,7 +186,7 @@ def _includeLiteral(self, content):
raise NotImplementedError()

def _resolveContent(self, result):
raise NotImplementedError()
return result

def resolve(self, result):
return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles)
Expand Down Expand Up @@ -397,20 +398,12 @@ def setupFingerprint(spec, env):


class PwshResolver(IncludeResolver):
def __init__(self, fileLoader, baseDir, origText, sourceName, varBase):
super().__init__(fileLoader, baseDir, origText, sourceName, varBase)
self.prolog = []
self.count = 0

def _includeFile(self, name):
return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"'

def _includeLiteral(self, content):
return quotePwsh(content.decode('utf8'))

def _resolveContent(self, result):
return "\n".join(self.prolog + [result])


class PwshLanguage:
index = ScriptLanguage.PWSH
Expand Down Expand Up @@ -574,9 +567,149 @@ def setupFingerprint(spec, env):
return [interpreter, "-c", spec.fingerprintScript]


class PythonResolver(IncludeResolver):
def _includeFile(self, name):
return 'os.path.join(_BOB_TMP_BASE, ' + repr(name) + ')'

def _includeLiteral(self, content):
return repr(content.decode('utf8'))


class PythonLanguage:
index = ScriptLanguage.PYTHON
glue = "\nos.chdir(os.environ['BOB_CWD'])\n"
Resolver = PythonResolver

PROLOGUE = dedent("""\
import os, os.path, sys
""")

@staticmethod
def __formatProlog(spec, tmpDir):
pathSep = ";" if sys.platform == "win32" else ":"
env = { key : repr(value) for (key, value) in spec.env.items() }
env.update({
"PATH": '"' + pathSep + '".join([' + ", ".join(
[repr(os.path.abspath(p)) for p in spec.paths] +
(['os.environ["PATH"]'] if not spec.hasSandbox else
[repr(p) for p in spec.sandboxPaths])
) + '])',
"LD_LIBRARY_PATH": '"' + pathSep + '".join(' +
repr([ os.path.abspath(p) for p in spec.libraryPaths ]) + ')',
"BOB_CWD": repr(os.path.abspath(spec.workspaceExecPath)),
})

ret = [
"# Convenience helpers",
PythonLanguage.PROLOGUE,
"",
"# Special Bob array variables:",
"BOB_ALL_PATHS = dict({})".format(repr(sorted(
[ (name, os.path.abspath(path)) for name, path in spec.allPaths ]
))),
"BOB_DEP_PATHS = dict({})".format(repr(sorted(
[ (name, os.path.abspath(path)) for name,path in spec.depPaths ]
))),
"BOB_TOOL_PATHS = dict({})".format(repr(sorted(
[ (name, os.path.abspath(path)) for name,path in spec.toolPaths ]
))),
'_BOB_TMP_BASE = ' + repr("/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp")),
"",
"# Environment:",
"\n".join('os.environ["{}"] = {}'.format(k, v) for (k,v) in sorted(env.items())),
]
return "\n".join(ret)

@staticmethod
def __formatSetup(spec):
return "\n".join([
"",
"# Recipe setup script",
spec.setupScript,
"os.chdir(os.environ['BOB_CWD'])",
])

@staticmethod
def __formatScript(spec, tmpDir, trace):
if spec.envFile:
envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile)
else:
envFile = None
ret = [
PythonLanguage.__formatProlog(spec, tmpDir),
"",
"# Setup",
dedent("""\
with open({ENV_FILE}, "w") as f:
f.write(repr({{ "env" : dict(os.environ), "globals" : globals() }}))
""".format(ENV_FILE=repr(envFile))) if envFile else "",
"os.chdir(os.environ['BOB_CWD'])",
"",
PythonLanguage.__formatSetup(spec),
"",
"# Recipe main script",
spec.mainScript,
]
return "\n".join(ret)

@staticmethod
def __scriptFilePaths(spec, tmpDir):
if spec.hasSandbox:
execScriptFile = "/.script.py"
realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".py"
else:
execScriptFile = (spec.scriptHint or os.path.join(tmpDir, "script")) + ".py"
realScriptFile = execScriptFile
return (os.path.abspath(realScriptFile), os.path.abspath(execScriptFile))

@staticmethod
def setupShell(spec, tmpDir, keepEnv):
realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
with open(realScriptFile, "w") as f:
f.write(PythonLanguage.__formatProlog(spec, tmpDir))
f.write(PythonLanguage.__formatSetup(spec))

args = [sys.executable, "-i", execScriptFile]
args.extend(os.path.abspath(a) for a in spec.args)

return (realScriptFile, execScriptFile, args)

@staticmethod
def setupCall(spec, tmpDir, keepEnv, trace):
realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
with open(realScriptFile, "w") as f:
f.write(PythonLanguage.__formatScript(spec, tmpDir, trace))

args = [sys.executable, execScriptFile]
args.extend(os.path.abspath(a) for a in spec.args)

return (realScriptFile, execScriptFile, args)

@staticmethod
def mangleFingerprints(scriptFragments, env):
# join the script fragments first
script = joinScripts(scriptFragments, PythonLanguage.glue)

# do not add preamble for empty scripts
if not script: return ""

# Add snippets as they match and a default settings preamble
ret = [script]
for n,v in sorted(env.items()):
ret.append('os.environ["{}"] = {}'.format(k, repr(v)))
ret.append(PythonLanguage.PROLOGUE)

return "\n".join(reversed(ret))

@staticmethod
def setupFingerprint(spec, env):
return [sys.executable, "-c", spec.fingerprintScript]


LANG = {
ScriptLanguage.BASH : BashLanguage,
ScriptLanguage.PWSH : PwshLanguage,
ScriptLanguage.PYTHON : PythonLanguage,
}

def getLanguage(language):
Expand Down

0 comments on commit 1920662

Please sign in to comment.