-
-
Notifications
You must be signed in to change notification settings - Fork 14.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
makeBinaryWrapper: create binary wrappers #95569
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
{ stdenv | ||
, targetPackages | ||
, python3Minimal | ||
, callPackage | ||
}: | ||
|
||
# Python is used for creating the instructions for the Nim program which | ||
# is compiled into a binary. | ||
# | ||
# The python3Minimal package is used because its available early on during bootstrapping. | ||
# Python packaging tools are avoided because this needs to be available early on in bootstrapping. | ||
|
||
let | ||
python = python3Minimal; | ||
sitePackages = "${placeholder "out"}/${python.sitePackages}"; | ||
nim = targetPackages.nim.override { | ||
minimal = true; | ||
}; | ||
in stdenv.mkDerivation { | ||
name = "make-binary-wrapper"; | ||
|
||
src = ./src; | ||
|
||
buildInputs = [ | ||
python | ||
]; | ||
|
||
strictDeps = true; | ||
|
||
postPatch = '' | ||
substituteInPlace lib/libwrapper/compile_wrapper.py \ | ||
--replace 'NIM_EXECUTABLE = "nim"' 'NIM_EXECUTABLE = "${nim}/bin/nim"' \ | ||
--replace 'STRIP_EXECUTABLE = "strip"' 'STRIP_EXECUTABLE = "${targetPackages.binutils-unwrapped}/bin/strip"' | ||
substituteAllInPlace bin/make-wrapper | ||
''; | ||
|
||
inherit sitePackages; | ||
|
||
dontBuild = true; | ||
|
||
installPhase = '' | ||
mkdir -p $out/${python.sitePackages} | ||
mv bin $out/ | ||
mv lib/libwrapper $out/${python.sitePackages} | ||
''; | ||
|
||
passthru.tests.test-wrapped-hello = callPackage ./tests.nix { | ||
inherit python; | ||
}; | ||
|
||
meta = { | ||
description = "Tool to create binary wrappers"; | ||
maintainers = with stdenv.lib.maintainers; [ fridh ]; | ||
}; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import sys | ||
sitepackages = "@sitePackages@" | ||
sys.path.insert(0, sitepackages) | ||
import libwrapper.make_wrapper | ||
libwrapper.make_wrapper.main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
make-wrapper |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
import json | ||
import pathlib | ||
import shutil | ||
import subprocess | ||
import tempfile | ||
from typing import Dict | ||
|
||
|
||
WRAPPER_SOURCE = pathlib.Path(__file__).parent / "wrapper.nim" | ||
|
||
NIM_EXECUTABLE = "nim" | ||
STRIP_EXECUTABLE = "strip" | ||
|
||
|
||
def compile_wrapper(instruction: Dict): | ||
"""Compile a wrapper using the given instruction.""" | ||
|
||
with tempfile.TemporaryDirectory() as tmpdir: | ||
tmpdir = pathlib.Path(tmpdir) | ||
shutil.copyfile(WRAPPER_SOURCE, tmpdir / "wrapper.nim" ) | ||
with open(tmpdir / "wrapper.json", "w") as fout: | ||
json.dump(instruction, fout) | ||
subprocess.run( | ||
f"cd {tmpdir} && {NIM_EXECUTABLE} --nimcache=. --gc:none -d:release --opt:size compile {tmpdir}/wrapper.nim && {STRIP_EXECUTABLE} -s {tmpdir}/wrapper", | ||
shell=True, | ||
) | ||
shutil.move(tmpdir / "wrapper", instruction["wrapper"]) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
import argparse | ||
import textwrap | ||
from typing import Dict | ||
|
||
import libwrapper.compile_wrapper | ||
|
||
|
||
EPILOG = textwrap.dedent('''\ | ||
This program creates a binary wrapper. The arguments given are | ||
serialized to JSON. A binary wrapper is created and the JSON is | ||
embedded into it. | ||
|
||
For debugging purposes it is possible to view the embedded JSON: | ||
|
||
NIX_DEBUG_PYTHON=1 my-wrapped-executable | ||
|
||
''') | ||
|
||
|
||
def parse_args() -> Dict: | ||
|
||
parser = argparse.ArgumentParser( | ||
description="Create a binary wrapper.", | ||
epilog=EPILOG, | ||
formatter_class=argparse.RawDescriptionHelpFormatter, | ||
) | ||
|
||
parser.add_argument("original", type=str, | ||
help="Path of executable to wrap", | ||
) | ||
parser.add_argument("wrapper", type=str, | ||
help="Path of wrapper to create", | ||
) | ||
# parser.add_argument( | ||
# "--argv", nargs=1, type=str, metavar="NAME", default, | ||
# help="Set name of executed process. Not used." | ||
# ) | ||
parser.add_argument( | ||
"--set", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[], | ||
help="Set environment variable to value", | ||
) | ||
parser.add_argument( | ||
"--set-default", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[], | ||
help="Set environment variable to value, if not yet set in environment", | ||
) | ||
parser.add_argument( | ||
"--unset", nargs=1, type=str, metavar="VAR", action="append", default=[], | ||
help="Unset variable from the environment" | ||
) | ||
parser.add_argument( | ||
"--run", nargs=1, type=str, metavar="COMMAND", action="append", | ||
help="Run command before the executable" | ||
) | ||
parser.add_argument( | ||
"--add-flags", dest="flags", nargs=1, type=str, metavar="FLAGS", action="append", default=[], | ||
help="Add flags to invocation of process" | ||
) | ||
parser.add_argument( | ||
"--prefix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[], | ||
help="Prefix environment variable ENV with value VAL, separated by separator SEP" | ||
) | ||
parser.add_argument( | ||
"--suffix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[], | ||
help="Suffix environment variable ENV with value VAL, separated by separator SEP" | ||
) | ||
# TODO: Fix help message because we cannot use metavar with nargs="+". | ||
# Note these hardly used in Nixpkgs and may as well be dropped. | ||
# parser.add_argument( | ||
# "--prefix-each", nargs="+", type=str, action="append", | ||
# help="Prefix environment variable ENV with values VALS, separated by separator SEP." | ||
# ) | ||
# parser.add_argument( | ||
# "--suffix-each", nargs="+", type=str, action="append", | ||
# help="Suffix environment variable ENV with values VALS, separated by separator SEP." | ||
# ) | ||
# parser.add_argument( | ||
# "--prefix-contents", nargs="+", type=str, action="append", | ||
# help="Prefix environment variable ENV with values read from FILES, separated by separator SEP." | ||
# ) | ||
# parser.add_argument( | ||
# "--suffix-contents", nargs="+", type=str, action="append", | ||
# help="Suffix environment variable ENV with values read from FILES, separated by separator SEP." | ||
# ) | ||
return vars(parser.parse_args()) | ||
|
||
|
||
def convert_args(args: Dict) -> Dict: | ||
"""Convert arguments to the JSON structure expected by the Nim wrapper.""" | ||
output = {} | ||
|
||
# Would not need this if the Environment members were part of Wrapper. | ||
output["original"] = args["original"] | ||
output["wrapper"] = args["wrapper"] | ||
output["run"] = args["run"] | ||
output["flags"] = [item[0] for item in args["flags"]] | ||
|
||
output["environment"] = {} | ||
for key, value in args.items(): | ||
if key == "set": | ||
output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value] | ||
if key == "set_default": | ||
output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value] | ||
if key == "unset": | ||
output["environment"][key] = [dict(zip(["variable"], item)) for item in value] | ||
if key == "prefix": | ||
output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value] | ||
if key == "suffix": | ||
output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value] | ||
|
||
return output | ||
|
||
|
||
def main(): | ||
args = parse_args() | ||
args = convert_args(args) | ||
libwrapper.compile_wrapper.compile_wrapper(args) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
# This is the code of the wrapper that is generated. | ||
|
||
import json | ||
import os | ||
import posix | ||
import sequtils | ||
import strutils | ||
|
||
# Wrapper type as used by the wrapper-generation code as well in the actual wrapper. | ||
|
||
type | ||
SetVar* = object | ||
variable*: string | ||
value*: string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd like it to be possible to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The way it is now you can have "environment": {
"set": [
{
"variable": "HELLO",
"value": "foo"
},
{
"variable": "HELLO",
"value": "bar"
}
]
} Thus, order will matter. Am I correct you are suggesting you want the following? "environment": {
"set": [
{
"variable": "HELLO",
"values": [
"foo",
"bar"
]
}
]
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes I want it to be an array, and also there should be a Am I correct to understand there's both There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I see what you're after. What I did here is basically take the |
||
|
||
SetDefaultVar* = object | ||
variable*: string | ||
value*: string | ||
|
||
UnsetVar* = object | ||
variable*: string | ||
|
||
PrefixVar* = object | ||
variable*: string | ||
values*: seq[string] | ||
separator*: string | ||
|
||
SuffixVar* = object | ||
variable*: string | ||
values*: seq[string] | ||
separator*: string | ||
|
||
# Maybe move the members into Wrapper directly? | ||
Environment* = object | ||
set*: seq[SetVar] | ||
set_default*: seq[SetDefaultVar] | ||
unset*: seq[UnsetVar] | ||
prefix*: seq[PrefixVar] | ||
suffix*: seq[SuffixVar] | ||
|
||
Wrapper* = object | ||
original*: string | ||
wrapper*: string | ||
run*: string | ||
flags*: seq[string] | ||
environment*: Environment | ||
|
||
# File containing wrapper instructions | ||
const jsonFilename = "./wrapper.json" | ||
|
||
# Embed the JSON string defining the wrapper in our binary | ||
const jsonString = staticRead(jsonFilename) | ||
|
||
proc modifyEnv(item: SetVar) = | ||
putEnv(item.variable, item.value) | ||
|
||
proc modifyEnv(item: SetDefaultVar) = | ||
if not existsEnv(item.variable): | ||
putEnv(item.variable, item.value) | ||
|
||
proc modifyEnv(item: UnsetVar) = | ||
if existsEnv(item.variable): | ||
delEnv(item.variable) | ||
|
||
proc modifyEnv(item: PrefixVar) = | ||
let old_value = if existsEnv(item.variable): getEnv(item.variable) else: "" | ||
let new_value = join(concat(item.values, @[old_value]), item.separator) | ||
putEnv(item.variable, new_value) | ||
|
||
proc modifyEnv(item: SuffixVar) = | ||
let old_value = if existsEnv(item.variable): getEnv(item.variable) else: "" | ||
let new_value = join(concat(@[old_value], item.values), item.separator) | ||
putEnv(item.variable, new_value) | ||
|
||
proc processEnvironment(environment: Environment) = | ||
for item in environment.unset.items(): | ||
item.modifyEnv() | ||
for item in environment.set.items(): | ||
item.modifyEnv() | ||
for item in environment.set_default.items(): | ||
item.modifyEnv() | ||
for item in environment.prefix.items(): | ||
item.modifyEnv() | ||
for item in environment.suffix.items(): | ||
item.modifyEnv() | ||
|
||
|
||
if existsEnv("NIX_DEBUG_WRAPPER"): | ||
echo(jsonString) | ||
else: | ||
# Unfortunately parsing JSON during compile-time is not supported. | ||
let wrapperDescription = parseJson(jsonString) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the Npeg library could parse the environment at compile time - https://github.com/zevv/npeg There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. … or just generate Nim code directly? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you elaborate what you mean with generating Nim code here directly? It's important that a generated wrapper does what it should do, but that we can also still check how it was configured. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I mean generating a |
||
let wrapper = to(wrapperDescription, Wrapper) | ||
processEnvironment(wrapper.environment) | ||
let argv = wrapper.original # convert target to cstring | ||
let argc = allocCStringArray(wrapper.flags) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. flags don't work yet |
||
|
||
# Run command in new environment but before executing our executable | ||
discard execShellCmd(wrapper.run) | ||
discard execvp(argv, argc) # Maybe use execvpe instead so we can pass an updated mapping? | ||
|
||
deallocCStringArray(argc) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
{ runCommand | ||
, makeBinaryWrapper | ||
, python | ||
, stdenv | ||
}: | ||
|
||
runCommand "test-wrapped-hello" { | ||
nativeBuildInputs = [ | ||
makeBinaryWrapper | ||
]; | ||
} ('' | ||
mkdir -p $out/bin | ||
|
||
# Test building of the wrapper. | ||
|
||
make-wrapper ${python.interpreter} $out/bin/python \ | ||
--set FOO bar \ | ||
--set-default BAR foo \ | ||
--prefix MYPATH ":" zero \ | ||
--suffix MYPATH ":" four \ | ||
--unset UNSET_THIS | ||
|
||
'' + stdenv.lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' | ||
# When not cross-compiling we can execute the wrapper and test how it behaves. | ||
|
||
# See the following tests for why variables are set the way they are. | ||
|
||
# Test `set`: We set FOO to bar | ||
$out/bin/python -c "import os; assert os.environ["FOO] == "bar" | ||
|
||
# Test `set-default`: We set BAR to bar, and then set-default BAR, thus expecting the original bar. | ||
export BAR=bar | ||
$out/bin/python -c "import os; assert os.environ["BAR] == "bar" | ||
|
||
# Test `unset`: # We set MYPATH and unset it in the wrapper. | ||
export UNSET_THIS=1 | ||
$out/bin/python -c "import os; assert "UNSET_THIS" not in os.environ.["BAR]" | ||
|
||
# Test `prefix`: | ||
export MYPATH=one:two:three | ||
$out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "zero" | ||
|
||
# Test `suffix`: | ||
$out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "four" | ||
|
||
# Test `NIX_DEBUG_WRAPPER`: | ||
NIX_DEBUG_WRAPPER=1 $out/bin/python | ||
'') |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The JSON created for a wrapper needs to follow this exact structure.