Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

makeBinaryWrapper: create binary wrappers #95569

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions pkgs/build-support/make-binary-wrapper/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{ stdenv
, targetPackages
, python3Minimal
, callPackage
}:

# Python is used for creating the instructions for the Nim program which
# is compiled into a binary.
#
# The python3Minimal package is used because its available early on during bootstrapping.
# Python packaging tools are avoided because this needs to be available early on in bootstrapping.

let
python = python3Minimal;
sitePackages = "${placeholder "out"}/${python.sitePackages}";
nim = targetPackages.nim.override {
minimal = true;
};
in stdenv.mkDerivation {
name = "make-binary-wrapper";

src = ./src;

buildInputs = [
python
];

strictDeps = true;

postPatch = ''
substituteInPlace lib/libwrapper/compile_wrapper.py \
--replace 'NIM_EXECUTABLE = "nim"' 'NIM_EXECUTABLE = "${nim}/bin/nim"' \
--replace 'STRIP_EXECUTABLE = "strip"' 'STRIP_EXECUTABLE = "${targetPackages.binutils-unwrapped}/bin/strip"'
substituteAllInPlace bin/make-wrapper
'';

inherit sitePackages;

dontBuild = true;

installPhase = ''
mkdir -p $out/${python.sitePackages}
mv bin $out/
mv lib/libwrapper $out/${python.sitePackages}
'';

passthru.tests.test-wrapped-hello = callPackage ./tests.nix {
inherit python;
};

meta = {
description = "Tool to create binary wrappers";
maintainers = with stdenv.lib.maintainers; [ fridh ];
};
}
7 changes: 7 additions & 0 deletions pkgs/build-support/make-binary-wrapper/src/bin/make-wrapper
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env python3

import sys
sitepackages = "@sitePackages@"
sys.path.insert(0, sitepackages)
import libwrapper.make_wrapper
libwrapper.make_wrapper.main()
1 change: 1 addition & 0 deletions pkgs/build-support/make-binary-wrapper/src/bin/makeWrapper
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import json
import pathlib
import shutil
import subprocess
import tempfile
from typing import Dict


WRAPPER_SOURCE = pathlib.Path(__file__).parent / "wrapper.nim"

NIM_EXECUTABLE = "nim"
STRIP_EXECUTABLE = "strip"


def compile_wrapper(instruction: Dict):
"""Compile a wrapper using the given instruction."""

with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = pathlib.Path(tmpdir)
shutil.copyfile(WRAPPER_SOURCE, tmpdir / "wrapper.nim" )
with open(tmpdir / "wrapper.json", "w") as fout:
json.dump(instruction, fout)
subprocess.run(
f"cd {tmpdir} && {NIM_EXECUTABLE} --nimcache=. --gc:none -d:release --opt:size compile {tmpdir}/wrapper.nim && {STRIP_EXECUTABLE} -s {tmpdir}/wrapper",
shell=True,
)
shutil.move(tmpdir / "wrapper", instruction["wrapper"])

Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
import argparse
import textwrap
from typing import Dict

import libwrapper.compile_wrapper


EPILOG = textwrap.dedent('''\
This program creates a binary wrapper. The arguments given are
serialized to JSON. A binary wrapper is created and the JSON is
embedded into it.

For debugging purposes it is possible to view the embedded JSON:

NIX_DEBUG_PYTHON=1 my-wrapped-executable

''')


def parse_args() -> Dict:

parser = argparse.ArgumentParser(
description="Create a binary wrapper.",
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)

parser.add_argument("original", type=str,
help="Path of executable to wrap",
)
parser.add_argument("wrapper", type=str,
help="Path of wrapper to create",
)
# parser.add_argument(
# "--argv", nargs=1, type=str, metavar="NAME", default,
# help="Set name of executed process. Not used."
# )
parser.add_argument(
"--set", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[],
help="Set environment variable to value",
)
parser.add_argument(
"--set-default", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[],
help="Set environment variable to value, if not yet set in environment",
)
parser.add_argument(
"--unset", nargs=1, type=str, metavar="VAR", action="append", default=[],
help="Unset variable from the environment"
)
parser.add_argument(
"--run", nargs=1, type=str, metavar="COMMAND", action="append",
help="Run command before the executable"
)
parser.add_argument(
"--add-flags", dest="flags", nargs=1, type=str, metavar="FLAGS", action="append", default=[],
help="Add flags to invocation of process"
)
parser.add_argument(
"--prefix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[],
help="Prefix environment variable ENV with value VAL, separated by separator SEP"
)
parser.add_argument(
"--suffix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[],
help="Suffix environment variable ENV with value VAL, separated by separator SEP"
)
# TODO: Fix help message because we cannot use metavar with nargs="+".
# Note these hardly used in Nixpkgs and may as well be dropped.
# parser.add_argument(
# "--prefix-each", nargs="+", type=str, action="append",
# help="Prefix environment variable ENV with values VALS, separated by separator SEP."
# )
# parser.add_argument(
# "--suffix-each", nargs="+", type=str, action="append",
# help="Suffix environment variable ENV with values VALS, separated by separator SEP."
# )
# parser.add_argument(
# "--prefix-contents", nargs="+", type=str, action="append",
# help="Prefix environment variable ENV with values read from FILES, separated by separator SEP."
# )
# parser.add_argument(
# "--suffix-contents", nargs="+", type=str, action="append",
# help="Suffix environment variable ENV with values read from FILES, separated by separator SEP."
# )
return vars(parser.parse_args())


def convert_args(args: Dict) -> Dict:
"""Convert arguments to the JSON structure expected by the Nim wrapper."""
output = {}

# Would not need this if the Environment members were part of Wrapper.
output["original"] = args["original"]
output["wrapper"] = args["wrapper"]
output["run"] = args["run"]
output["flags"] = [item[0] for item in args["flags"]]

output["environment"] = {}
for key, value in args.items():
if key == "set":
output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value]
if key == "set_default":
output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value]
if key == "unset":
output["environment"][key] = [dict(zip(["variable"], item)) for item in value]
if key == "prefix":
output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value]
if key == "suffix":
output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value]

return output


def main():
args = parse_args()
args = convert_args(args)
libwrapper.compile_wrapper.compile_wrapper(args)


if __name__ == "__main__":
main()
102 changes: 102 additions & 0 deletions pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/wrapper.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# This is the code of the wrapper that is generated.

import json
import os
import posix
import sequtils
import strutils

# Wrapper type as used by the wrapper-generation code as well in the actual wrapper.

type
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The JSON created for a wrapper needs to follow this exact structure.

SetVar* = object
variable*: string
value*: string
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like it to be possible to SetVar multiple values from an array in wrapper.json.

Copy link
Member Author

@FRidh FRidh Aug 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way it is now you can have

    "environment": {
        "set": [
            {
                "variable": "HELLO",
                "value": "foo"
            },
            {
                "variable": "HELLO",
                "value": "bar"
            }
        ]
    }

Thus, order will matter.

Am I correct you are suggesting you want the following?

    "environment": {
        "set": [
            {
                "variable": "HELLO",
                "values": [ 
                    "foo",
                    "bar"
                ]
            }
        ]
    }

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I want it to be an array, and also there should be a separator key as well, because some env vars use e.g ; and not :. : should be the default though.

Am I correct to understand there's both SetVar and environment.set?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

environment.set is a sequence of SetVar objects where SetVar has the fields variable and value.

I see what you're after. What I did here is basically take the makeWrapper arguments as is. If you want export HELLO="foo:bar" then in case of makeWrapper write --set HELLO "foo:bar", and so you have variable HELLO and value foo:bar. In that case one does not need to know what separator is used.


SetDefaultVar* = object
variable*: string
value*: string

UnsetVar* = object
variable*: string

PrefixVar* = object
variable*: string
values*: seq[string]
separator*: string

SuffixVar* = object
variable*: string
values*: seq[string]
separator*: string

# Maybe move the members into Wrapper directly?
Environment* = object
set*: seq[SetVar]
set_default*: seq[SetDefaultVar]
unset*: seq[UnsetVar]
prefix*: seq[PrefixVar]
suffix*: seq[SuffixVar]

Wrapper* = object
original*: string
wrapper*: string
run*: string
flags*: seq[string]
environment*: Environment

# File containing wrapper instructions
const jsonFilename = "./wrapper.json"

# Embed the JSON string defining the wrapper in our binary
const jsonString = staticRead(jsonFilename)

proc modifyEnv(item: SetVar) =
putEnv(item.variable, item.value)

proc modifyEnv(item: SetDefaultVar) =
if not existsEnv(item.variable):
putEnv(item.variable, item.value)

proc modifyEnv(item: UnsetVar) =
if existsEnv(item.variable):
delEnv(item.variable)

proc modifyEnv(item: PrefixVar) =
let old_value = if existsEnv(item.variable): getEnv(item.variable) else: ""
let new_value = join(concat(item.values, @[old_value]), item.separator)
putEnv(item.variable, new_value)

proc modifyEnv(item: SuffixVar) =
let old_value = if existsEnv(item.variable): getEnv(item.variable) else: ""
let new_value = join(concat(@[old_value], item.values), item.separator)
putEnv(item.variable, new_value)

proc processEnvironment(environment: Environment) =
for item in environment.unset.items():
item.modifyEnv()
for item in environment.set.items():
item.modifyEnv()
for item in environment.set_default.items():
item.modifyEnv()
for item in environment.prefix.items():
item.modifyEnv()
for item in environment.suffix.items():
item.modifyEnv()


if existsEnv("NIX_DEBUG_WRAPPER"):
echo(jsonString)
else:
# Unfortunately parsing JSON during compile-time is not supported.
let wrapperDescription = parseJson(jsonString)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the Npeg library could parse the environment at compile time - https://github.com/zevv/npeg

Copy link
Contributor

@ehmry ehmry Aug 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

… or just generate Nim code directly?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you elaborate what you mean with generating Nim code here directly? It's important that a generated wrapper does what it should do, but that we can also still check how it was configured.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean generating a const Nim tuple of arrays rather than JSON. I was just thinking that the wrapper shouldn't neeed to do much heap allocation.

let wrapper = to(wrapperDescription, Wrapper)
processEnvironment(wrapper.environment)
let argv = wrapper.original # convert target to cstring
let argc = allocCStringArray(wrapper.flags)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

flags don't work yet


# Run command in new environment but before executing our executable
discard execShellCmd(wrapper.run)
discard execvp(argv, argc) # Maybe use execvpe instead so we can pass an updated mapping?

deallocCStringArray(argc)
48 changes: 48 additions & 0 deletions pkgs/build-support/make-binary-wrapper/tests.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{ runCommand
, makeBinaryWrapper
, python
, stdenv
}:

runCommand "test-wrapped-hello" {
nativeBuildInputs = [
makeBinaryWrapper
];
} (''
mkdir -p $out/bin

# Test building of the wrapper.

make-wrapper ${python.interpreter} $out/bin/python \
--set FOO bar \
--set-default BAR foo \
--prefix MYPATH ":" zero \
--suffix MYPATH ":" four \
--unset UNSET_THIS

'' + stdenv.lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) ''
# When not cross-compiling we can execute the wrapper and test how it behaves.

# See the following tests for why variables are set the way they are.

# Test `set`: We set FOO to bar
$out/bin/python -c "import os; assert os.environ["FOO] == "bar"

# Test `set-default`: We set BAR to bar, and then set-default BAR, thus expecting the original bar.
export BAR=bar
$out/bin/python -c "import os; assert os.environ["BAR] == "bar"

# Test `unset`: # We set MYPATH and unset it in the wrapper.
export UNSET_THIS=1
$out/bin/python -c "import os; assert "UNSET_THIS" not in os.environ.["BAR]"

# Test `prefix`:
export MYPATH=one:two:three
$out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "zero"

# Test `suffix`:
$out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "four"

# Test `NIX_DEBUG_WRAPPER`:
NIX_DEBUG_WRAPPER=1 $out/bin/python
'')
Loading